{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym, random, pickle, os.path, math, glob\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.autograd as autograd\n",
    "import pdb\n",
    "\n",
    "from atari_wrappers import make_atari, wrap_deepmind,LazyFrames\n",
    "from IPython.display import clear_output\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1811b5cb588>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPCklEQVR4nO3da6wc5X3H8e/fx5eDbcCXAnVthE1EuZQIQ48IlKoiEBpKUohaUoGiKq2QeJOm0EQK0L6IIlUqkaoEXkRVLUiKqoSbAw2yIqhl4EWl1mAuIYDtYAOFAw52AENibIPtf1/M2D5y93DmnL2cXT/fj3S0M7Oznmc8+u0zOzv7/CMzkXT0mzHdDZDUG4ZdKoRhlwph2KVCGHapEIZdKkRbYY+IyyNic0RsiYibO9UoSZ0XU/2ePSKGgF8AlwGjwJPAtZn5YueaJ6lTZrbx2vOBLZn5MkBE3ANcBYwb9tkxJ4eZN/G/PHf48PSMaKOJnZN1O/Yd0157Zu6p3lxjvzczdVuJx2zPnp18+NGuljvcTtiXAq+PmR8FPvVxLxhmHp+KSyf8h+Os3zs0vX/+7Ck2r7P2zxkCYMc57bVn8QsfATBr176226SPd/CY/eqT7R2zRRsH55g9ueF74z7XTthbvXv8v7e+iLgeuB5gmLltbE5SO9oJ+yhw8pj5ZcCbR66UmauAVQDHHbs0D4yc28Ymp8+Boeq9bdcp+9v6dxZsra6JztrVdpM0gYPH7DfL2ztmx79ydByzdq7GPwmcFhErImI2cA3wUGeaJanTptyzZ+a+iPgb4BFgCPh+Zr7QsZb1mVkfVJ/XPnF/89e88qdzADgw3P8XdkrzidUftlz+6ueqi8P7jznQy+b0RDun8WTmT4GfdqgtkrrIO+ikQrTVs5fkwFD1vvjeiuZf4+SQp+/qH/bsUiHs2RvaP6d6X3zn3KPvwo3KYM8uFcKwS4XwNL4Dlj7afN3ZO/v//modnezZpUIYdqkQnsarSHsWt75fImccvfdG2LNLhbBn74A3Lmm+7tJHq//y4bdb/xBDvfHGp8d7xp5d0oAz7FIhPI1vaGhvdZvsomfa+y+bueujTjRHDRw6Zs96zMCeXSpGT3v2vQtnsPXq/hgtdura+yHMOyuH6qmhj11PnVTOMdu7dfz+e8KePSK+HxHbI+L5McsWRcTaiHipflzYobZK6pImp/H/Blx+xLKbgXWZeRqwrp6X1McalX+KiOXAmsw8u57fDFycmdsiYgnweGaePtG/M3LOcD7xyMkTrSZpis7/7Ots+NmelhVhpnqB7qTM3AZQP5441cZJ6o2uX42PiOsjYkNEbNjxdnuD9UuauqmG/a369J36cft4K2bmqswcycyRExb3/9VM6Wg11bA/BHy5nv4y8JPONEdStzT56u1u4L+B0yNiNCKuA24FLouIl6jqs9/a3WZKateEN9Vk5rXjPDVx7WVJfcPbZaVC9PR22U27F3DRc3/Wy01K0+qUY989NH3a/MPXsUd3VzedbtrZ2W+tN+2+a9zn7NmlQvS0Z5+xbYi5/3R8LzcpTauNZ//2oen/OeN3D03Pf7X6GvqkDbs7ur0Z28b/etueXSqEYZcKYdilQhh2qRCGXSqEA05KXTS05/B4EbN2Hu5bhzp7Eb4Re3apEPbsUhct3LJnzPQ0NgR7dqkYhl0qhGGXCmHYpUIYdqkQTYalOjkiHouIjRHxQkTcUC+3Kow0QJr07PuAr2fmmcAFwFci4iysCiMNlAnDnpnbMvPpevrXwEZgKXAVcHBYjLuAL3SrkZLaN6nP7HUZqHOB9TSsCjO2SMSHH+1qr7WSpqxx2CNiPvBj4MbMfL/p68YWiZg9a95U2iipAxqFPSJmUQX9h5n5QL24cVUYSdOvydX4AO4ENmbmd8Y8ZVUYaYA0+SHMRcBfAj+PiGfrZX9PVQXmvrpCzGvAF7vTREmd0KQizH8BLes9Y1UYaWB4B51UCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUiCZj0A1HxBMR8bO6Isy36uUrImJ9XRHm3oiY3f3mSpqqJj37XuCSzDwHWAlcHhEXAN8GvltXhHkXuK57zZTUriYVYTIzf1PPzqr/ErgEWF0vtyKM1Oeajhs/VI8sux1YC2wFdmbmvnqVUaqSUK1ea0UYqQ80Cntm7s/MlcAy4HzgzFarjfNaK8JIfWBSV+MzcyfwOFU11wURcXAo6mXAm51tmqROanI1/oSIWFBPHwN8hqqS62PA1fVqVoSR+lyTijBLgLsiYojqzeG+zFwTES8C90TEPwLPUJWIktSnmlSEeY6qTPORy1+m+vwuaQB4B51UCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUiMZhr4eTfiYi1tTzVoSRBshkevYbqAaaPMiKMNIAaVokYhnwOeCOej6wIow0UJr27LcB3wAO1POLsSKMNFCajBv/eWB7Zj41dnGLVa0II/WxJuPGXwRcGRFXAMPAcVQ9/YKImFn37laEkfpckyqut2TmssxcDlwDPJqZX8KKMNJAaed79puAr0XEFqrP8FaEkfpYk9P4QzLzcarCjlaEkQaMd9BJhTDsUiEMu1QIwy4VwrBLhTDsUiEMu1QIwy4VwrBLhTDsUiEMu1QIwy4VwrBLhTDsUiEMu1QIwy4VwrBLhWg0Uk1EvAr8GtgP7MvMkYhYBNwLLAdeBf4iM9/tTjMltWsyPfunM3NlZo7U8zcD6+qKMOvqeUl9qp3T+KuoKsGAFWGkvtc07An8Z0Q8FRHX18tOysxtAPXjia1eaEUYqT80HV32osx8MyJOBNZGxKamG8jMVcAqgOOOXdqyaoyk7mvUs2fmm/XjduBBqiGk34qIJQD14/ZuNVJS+5rUepsXEccenAb+GHgeeIiqEgxYEUbqe01O408CHqyqNDMT+FFmPhwRTwL3RcR1wGvAF7vXTEntmjDsdeWXc1osfxu4tBuNktR53kEnFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFaJR2CNiQUSsjohNEbExIi6MiEURsTYiXqofF3a7sZKmrmnPfjvwcGaeQTVE1UasCCMNlAnHoIuI44A/Av4KIDM/BD6MiKuAi+vV7gIeB27qRiM/znsrhgHYfUIcWjb/jQNjpvf2uklSX2rSs58K7AB+EBHPRMQd9ZDSVoSRBkiToaRnAucBX83M9RFxO5M4Ze92RZiDPfquU/YfWjZz99Ch6flvdHqL0mBq0rOPAqOZub6eX00VfivCSANkwrBn5i+B1yPi9HrRpcCLWBFGGihNCzt+FfhhRMwGXgb+muqNwoow0oBoFPbMfBYYafGUFWGkAeEddFIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFaLpD2H61jE7Dv5E/vBv2IffOdB6Zalg9uxSIQa+Zz/+lT314zQ3ROpz9uxSIQy7VIgJwx4Rp0fEs2P+3o+IGy0SIQ2WJmPQbc7MlZm5Evh94APgQSwSIQ2UyZ7GXwpszcz/Ba6iKg5B/fiFTjZMUmdNNuzXAHfX042KREjqD43DXo8seyVw/2Q2YEUYqT9Mpmf/E+DpzHyrnm9UJCIzV2XmSGaOzJ41r73WSpqyyYT9Wg6fwoNFIqSB0rQ++1zgMuCBMYtvBS6LiJfq527tfPMkdUrTIhEfAIuPWPY2FomQBoZ30EmFMOxSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhWip+PG7104g61Xz+7lJqWi7N06fv9tzy4Voqc9+ycX7uCJP//XXm5SKsr5q3aM+5w9u1QIwy4VoumwVH8XES9ExPMRcXdEDEfEiohYX1eEubcefVZSn2pS/mkp8LfASGaeTVUI/Rrg28B364ow7wLXdbOhktrT9DR+JnBMRMwE5gLbgEuA1fXzVoSR+lyTWm9vAP8MvEYV8veAp4CdmbmvXm0UWNqtRkpqX5PT+IVUdd1WAL8DzKMqGHGkHOf1hyrC7Hh7fzttldSGJqfxnwFeycwdmfkR1djxfwAsqE/rAZYBb7Z68diKMCcsHupIoyVNXpOwvwZcEBFzIyKoxop/EXgMuLpex4owUp9r8pl9PdWFuKeBn9evWQXcBHwtIrZQFZC4s4vtlNSmphVhvgl884jFLwPnd7xFkrrCO+ikQhh2qRCGXSqEYZcKEZkt74XpzsYidgC7gF/1bKPd91u4P/3qaNoXaLY/p2TmCa2e6GnYASJiQ2aO9HSjXeT+9K+jaV+g/f3xNF4qhGGXCjEdYV81DdvsJvenfx1N+wJt7k/PP7NLmh6exkuF6GnYI+LyiNgcEVsi4uZebrtdEXFyRDwWERvr8fhuqJcvioi19Vh8a+vf/w+MiBiKiGciYk09P7BjC0bEgohYHRGb6uN04SAfn06P/dizsEfEEPA9qoEvzgKujYizerX9DtgHfD0zzwQuAL5St/9mYF09Ft+6en6Q3ABsHDM/yGML3g48nJlnAOdQ7ddAHp+ujP2YmT35Ay4EHhkzfwtwS6+234X9+QlwGbAZWFIvWwJsnu62TWIfllEF4BJgDRBUN23MbHXM+vkPOA54hfo61JjlA3l8qIZ5ex1YRPXr1DXAZ9s5Pr08jT/Y+IMGdty6iFgOnAusB07KzG0A9eOJ09eySbsN+AZwoJ5fzOCOLXgqsAP4Qf2x5I6ImMeAHp/swtiPvQx7tFg2cF8FRMR84MfAjZn5/nS3Z6oi4vPA9sx8auziFqsOyjGaCZwH/Etmnkt1W/ZAnLK30u7Yj630MuyjwMlj5scdt65fRcQsqqD/MDMfqBe/FRFL6ueXANunq32TdBFwZUS8CtxDdSp/Gw3HFuxDo8BoViMrQTW60nkM7vFpa+zHVnoZ9ieB0+qribOpLjY81MPtt6Uef+9OYGNmfmfMUw9RjcEHAzQWX2bekpnLMnM51bF4NDO/xICOLZiZvwRej4jT60UHx0ocyONDN8Z+7PFFhyuAXwBbgX+Y7osgk2z7H1KdMj0HPFv/XUH1OXcd8FL9uGi62zqFfbsYWFNPnwo8AWwB7gfmTHf7JrEfK4EN9TH6D2DhIB8f4FvAJuB54N+BOe0cH++gkwrhHXRSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuF+D+ArU0D803TpAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create and wrap the environment\n",
    "env = make_atari('PongNoFrameskip-v4') # only use in no frameskip environment\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True )\n",
    "n_actions = env.action_space.n\n",
    "state_dim = env.observation_space.shape\n",
    "\n",
    "# env.render()\n",
    "test = env.reset()\n",
    "for i in range(100):\n",
    "    test = env.step(env.action_space.sample())[0]\n",
    "\n",
    "plt.imshow(test._force()[...,0])\n",
    "\n",
    "#plt.imshow(env.render(\"rgb_array\"))\n",
    "# env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, in_channels=4, num_actions=5):\n",
    "        \"\"\"\n",
    "        Initialize a deep Q-learning network as described in\n",
    "        https://storage.googleapis.com/deepmind-data/assets/papers/DeepMindNature14236Paper.pdf\n",
    "        Arguments:\n",
    "            in_channels: number of channel of input.\n",
    "                i.e The number of most recent frames stacked together as describe in the paper\n",
    "            num_actions: number of action-value to output, one-to-one correspondence to action in game.\n",
    "        \"\"\"\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.fc4 = nn.Linear(7 * 7 * 64, 512)\n",
    "        self.fc5 = nn.Linear(512, num_actions)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.fc4(x.view(x.size(0), -1)))\n",
    "        return self.fc5(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Memory_Buffer(object):\n",
    "    def __init__(self, memory_size=1000):\n",
    "        self.buffer = []\n",
    "        self.memory_size = memory_size\n",
    "        self.next_idx = 0\n",
    "        \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        data = (state, action, reward, next_state, done)\n",
    "        if len(self.buffer) <= self.memory_size: # buffer not full\n",
    "            self.buffer.append(data)\n",
    "        else: # buffer is full\n",
    "            self.buffer[self.next_idx] = data\n",
    "        self.next_idx = (self.next_idx + 1) % self.memory_size\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            idx = random.randint(0, self.size() - 1)\n",
    "            data = self.buffer[idx]\n",
    "            state, action, reward, next_state, done= data\n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(next_state)\n",
    "            dones.append(done)\n",
    "            \n",
    "            \n",
    "        return np.concatenate(states), actions, rewards, np.concatenate(next_states), dones\n",
    "    \n",
    "    def size(self):\n",
    "        return len(self.buffer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DDQNAgent: \n",
    "    def __init__(self, in_channels = 1, action_space = [], USE_CUDA = False, memory_size = 10000, epsilon  = 1, lr = 1e-4):\n",
    "        self.epsilon = epsilon\n",
    "        self.action_space = action_space\n",
    "        self.memory_buffer = Memory_Buffer(memory_size)\n",
    "        self.DQN = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target.load_state_dict(self.DQN.state_dict())\n",
    "\n",
    "\n",
    "        self.USE_CUDA = USE_CUDA\n",
    "        if USE_CUDA:\n",
    "            self.DQN = self.DQN.cuda()\n",
    "            self.DQN_target = self.DQN_target.cuda()\n",
    "        self.optimizer = optim.RMSprop(self.DQN.parameters(),lr=lr, eps=0.001, alpha=0.95)\n",
    "\n",
    "    def observe(self, lazyframe):\n",
    "        # from Lazy frame to tensor\n",
    "        state =  torch.from_numpy(lazyframe._force().transpose(2,0,1)[None]/255).float()\n",
    "        if self.USE_CUDA:\n",
    "            state = state.cuda()\n",
    "        return state\n",
    "\n",
    "    def value(self, state):\n",
    "        q_values = self.DQN(state)\n",
    "        return q_values\n",
    "    \n",
    "    def act(self, state, epsilon = None):\n",
    "        \"\"\"\n",
    "        sample actions with epsilon-greedy policy\n",
    "        recap: with p = epsilon pick random action, else pick action with highest Q(s,a)\n",
    "        \"\"\"\n",
    "        if epsilon is None: epsilon = self.epsilon\n",
    "\n",
    "        q_values = self.value(state).cpu().detach().numpy()\n",
    "        if random.random()<epsilon:\n",
    "            aciton = random.randrange(self.action_space.n)\n",
    "        else:\n",
    "            aciton = q_values.argmax(1)[0]\n",
    "        return aciton\n",
    "    \n",
    "    def compute_td_loss(self, states, actions, rewards, next_states, is_done, gamma=0.99):\n",
    "        \"\"\" Compute td loss using torch operations only. Use the formula above. \"\"\"\n",
    "        actions = torch.tensor(actions).long()    # shape: [batch_size]\n",
    "        rewards = torch.tensor(rewards, dtype =torch.float)  # shape: [batch_size]\n",
    "        is_done = torch.tensor(is_done).bool()  # shape: [batch_size]\n",
    "        \n",
    "        if self.USE_CUDA:\n",
    "            actions = actions.cuda()\n",
    "            rewards = rewards.cuda()\n",
    "            is_done = is_done.cuda()\n",
    "\n",
    "        # get q-values for all actions in current states\n",
    "        predicted_qvalues = self.DQN(states)\n",
    "\n",
    "        # select q-values for chosen actions\n",
    "        predicted_qvalues_for_actions = predicted_qvalues[\n",
    "          range(states.shape[0]), actions\n",
    "        ]\n",
    "\n",
    "        # compute q-values for all actions in next states \n",
    "        ## Where DDQN is different from DQN\n",
    "        predicted_next_qvalues_current = self.DQN(next_states)\n",
    "        predicted_next_qvalues_target = self.DQN_target(next_states)\n",
    "        # compute V*(next_states) using predicted next q-values\n",
    "        next_state_values =  predicted_next_qvalues_target.gather(1, torch.max(predicted_next_qvalues_current, 1)[1].unsqueeze(1)).squeeze(1)\n",
    "        \n",
    "        # compute \"target q-values\" for loss - it's what's inside square parentheses in the above formula.\n",
    "        target_qvalues_for_actions = rewards + gamma *next_state_values # YOUR CODE\n",
    "\n",
    "        # at the last state we shall use simplified formula: Q(s,a) = r(s,a) since s' doesn't exist\n",
    "        target_qvalues_for_actions = torch.where(\n",
    "            is_done, rewards, target_qvalues_for_actions)\n",
    "\n",
    "        # mean squared error loss to minimize\n",
    "        #loss = torch.mean((predicted_qvalues_for_actions -\n",
    "        #                   target_qvalues_for_actions.detach()) ** 2)\n",
    "        loss = F.smooth_l1_loss(predicted_qvalues_for_actions, target_qvalues_for_actions.detach())\n",
    "\n",
    "        return loss\n",
    "    \n",
    "    def sample_from_buffer(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            idx = random.randint(0, self.memory_buffer.size() - 1)\n",
    "            data = self.memory_buffer.buffer[idx]\n",
    "            frame, action, reward, next_frame, done= data\n",
    "            states.append(self.observe(frame))\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(self.observe(next_frame))\n",
    "            dones.append(done)\n",
    "        return torch.cat(states), actions, rewards, torch.cat(next_states), dones\n",
    "\n",
    "    def learn_from_experience(self, batch_size):\n",
    "        if self.memory_buffer.size() > batch_size:\n",
    "            states, actions, rewards, next_states, dones = self.sample_from_buffer(batch_size)\n",
    "            td_loss = self.compute_td_loss(states, actions, rewards, next_states, dones)\n",
    "            self.optimizer.zero_grad()\n",
    "            td_loss.backward()\n",
    "            for param in self.DQN.parameters():\n",
    "                param.grad.data.clamp_(-1, 1)\n",
    "\n",
    "            self.optimizer.step()\n",
    "            return(td_loss.item())\n",
    "        else:\n",
    "            return(0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\softwares\\ANACONDA\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "F:\\softwares\\ANACONDA\\lib\\site-packages\\numpy\\core\\_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "WARNING:root:NaN or Inf found in input tensor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames:     0, reward:   nan, loss: 0.000000, epsilon: 1.000000, episode:    0\n",
      "frames:  1000, reward: -20.000000, loss: 0.000000, epsilon: 0.967544, episode:    1\n",
      "frames:  2000, reward: -19.500000, loss: 0.000000, epsilon: 0.936152, episode:    2\n",
      "frames:  3000, reward: -19.666667, loss: 0.000000, epsilon: 0.905789, episode:    3\n",
      "frames:  4000, reward: -20.000000, loss: 0.000000, epsilon: 0.876422, episode:    4\n",
      "frames:  5000, reward: -19.600000, loss: 0.000000, epsilon: 0.848017, episode:    5\n",
      "frames:  6000, reward: -19.833333, loss: 0.000000, epsilon: 0.820543, episode:    6\n",
      "frames:  7000, reward: -19.571429, loss: 0.000000, epsilon: 0.793971, episode:    7\n",
      "frames:  8000, reward: -19.750000, loss: 0.000000, epsilon: 0.768269, episode:    8\n",
      "frames:  9000, reward: -19.666667, loss: 0.000000, epsilon: 0.743410, episode:    9\n",
      "frames: 10000, reward: -19.800000, loss: 0.000375, epsilon: 0.719366, episode:   10\n",
      "frames: 11000, reward: -19.900000, loss: 0.000353, epsilon: 0.696110, episode:   11\n",
      "frames: 12000, reward: -19.900000, loss: 0.015069, epsilon: 0.673617, episode:   12\n",
      "frames: 13000, reward: -19.800000, loss: 0.015388, epsilon: 0.651861, episode:   13\n",
      "frames: 14000, reward: -19.800000, loss: 0.000479, epsilon: 0.630818, episode:   14\n",
      "frames: 15000, reward: -20.100000, loss: 0.030820, epsilon: 0.610465, episode:   15\n",
      "frames: 16000, reward: -19.800000, loss: 0.044443, epsilon: 0.590780, episode:   16\n",
      "frames: 17000, reward: -20.000000, loss: 0.014967, epsilon: 0.571740, episode:   17\n",
      "frames: 18000, reward: -20.200000, loss: 0.000384, epsilon: 0.553324, episode:   19\n",
      "frames: 19000, reward: -20.200000, loss: 0.015398, epsilon: 0.535511, episode:   20\n",
      "frames: 20000, reward: -20.200000, loss: 0.000309, epsilon: 0.518283, episode:   21\n",
      "frames: 21000, reward: -20.400000, loss: 0.015430, epsilon: 0.501619, episode:   22\n",
      "frames: 22000, reward: -20.600000, loss: 0.000151, epsilon: 0.485502, episode:   23\n",
      "frames: 23000, reward: -20.600000, loss: 0.000093, epsilon: 0.469913, episode:   24\n",
      "frames: 24000, reward: -20.300000, loss: 0.015579, epsilon: 0.454836, episode:   25\n",
      "frames: 25000, reward: -20.600000, loss: 0.000222, epsilon: 0.440252, episode:   26\n",
      "frames: 26000, reward: -20.500000, loss: 0.015696, epsilon: 0.426147, episode:   27\n",
      "frames: 27000, reward: -20.500000, loss: 0.015159, epsilon: 0.412504, episode:   28\n",
      "frames: 28000, reward: -20.500000, loss: 0.000176, epsilon: 0.399308, episode:   29\n",
      "frames: 29000, reward: -20.400000, loss: 0.031375, epsilon: 0.386545, episode:   30\n",
      "frames: 30000, reward: -20.300000, loss: 0.015285, epsilon: 0.374201, episode:   31\n",
      "frames: 31000, reward: -20.200000, loss: 0.015154, epsilon: 0.362261, episode:   32\n",
      "frames: 32000, reward: -20.000000, loss: 0.029315, epsilon: 0.350712, episode:   34\n",
      "frames: 33000, reward: -20.300000, loss: 0.014687, epsilon: 0.339542, episode:   35\n",
      "frames: 34000, reward: -20.200000, loss: 0.015763, epsilon: 0.328739, episode:   36\n",
      "frames: 35000, reward: -20.400000, loss: 0.015578, epsilon: 0.318289, episode:   37\n",
      "frames: 36000, reward: -20.400000, loss: 0.016941, epsilon: 0.308182, episode:   38\n",
      "frames: 37000, reward: -20.400000, loss: 0.030010, epsilon: 0.298407, episode:   39\n",
      "frames: 38000, reward: -20.600000, loss: 0.000146, epsilon: 0.288952, episode:   41\n",
      "frames: 39000, reward: -20.700000, loss: 0.000191, epsilon: 0.279806, episode:   42\n",
      "frames: 40000, reward: -20.800000, loss: 0.000062, epsilon: 0.270961, episode:   43\n",
      "frames: 41000, reward: -20.700000, loss: 0.000064, epsilon: 0.262406, episode:   44\n",
      "frames: 42000, reward: -20.700000, loss: 0.000295, epsilon: 0.254131, episode:   45\n",
      "frames: 43000, reward: -20.700000, loss: 0.000974, epsilon: 0.246127, episode:   46\n",
      "frames: 44000, reward: -20.700000, loss: 0.000009, epsilon: 0.238386, episode:   47\n",
      "frames: 45000, reward: -20.700000, loss: 0.015151, epsilon: 0.230899, episode:   48\n",
      "frames: 46000, reward: -20.700000, loss: 0.000142, epsilon: 0.223657, episode:   49\n",
      "frames: 47000, reward: -20.500000, loss: 0.015559, epsilon: 0.216652, episode:   50\n",
      "frames: 48000, reward: -20.300000, loss: 0.031452, epsilon: 0.209878, episode:   52\n",
      "frames: 49000, reward: -20.400000, loss: 0.000639, epsilon: 0.203325, episode:   53\n",
      "frames: 50000, reward: -20.400000, loss: 0.030853, epsilon: 0.196987, episode:   54\n",
      "frames: 51000, reward: -20.400000, loss: 0.000223, epsilon: 0.190857, episode:   55\n",
      "frames: 52000, reward: -20.400000, loss: 0.000861, epsilon: 0.184928, episode:   56\n",
      "frames: 53000, reward: -20.400000, loss: 0.000181, epsilon: 0.179193, episode:   57\n",
      "frames: 54000, reward: -20.400000, loss: 0.029879, epsilon: 0.173646, episode:   58\n",
      "frames: 55000, reward: -20.300000, loss: 0.000657, epsilon: 0.168281, episode:   59\n",
      "frames: 56000, reward: -20.500000, loss: 0.000363, epsilon: 0.163092, episode:   60\n",
      "frames: 57000, reward: -20.500000, loss: 0.043827, epsilon: 0.158073, episode:   61\n",
      "frames: 58000, reward: -20.500000, loss: 0.014546, epsilon: 0.153219, episode:   62\n",
      "frames: 59000, reward: -20.400000, loss: 0.013332, epsilon: 0.148523, episode:   64\n",
      "frames: 60000, reward: -20.400000, loss: 0.020091, epsilon: 0.143982, episode:   64\n",
      "frames: 61000, reward: -20.200000, loss: 0.000967, epsilon: 0.139589, episode:   66\n",
      "frames: 62000, reward: -20.200000, loss: 0.000353, epsilon: 0.135341, episode:   66\n",
      "frames: 63000, reward: -20.000000, loss: 0.009848, epsilon: 0.131232, episode:   68\n",
      "frames: 64000, reward: -20.000000, loss: 0.001025, epsilon: 0.127257, episode:   69\n",
      "frames: 65000, reward: -19.900000, loss: 0.005357, epsilon: 0.123413, episode:   70\n",
      "frames: 66000, reward: -19.900000, loss: 0.006402, epsilon: 0.119695, episode:   71\n",
      "frames: 67000, reward: -20.000000, loss: 0.001422, epsilon: 0.116099, episode:   72\n",
      "frames: 68000, reward: -20.100000, loss: 0.001608, epsilon: 0.112621, episode:   73\n",
      "frames: 69000, reward: -20.100000, loss: 0.002712, epsilon: 0.109256, episode:   74\n",
      "frames: 70000, reward: -20.300000, loss: 0.001358, epsilon: 0.106002, episode:   75\n",
      "frames: 71000, reward: -20.300000, loss: 0.001959, epsilon: 0.102855, episode:   76\n",
      "frames: 72000, reward: -20.500000, loss: 0.001757, epsilon: 0.099811, episode:   78\n",
      "frames: 73000, reward: -20.500000, loss: 0.001165, epsilon: 0.096866, episode:   78\n",
      "frames: 74000, reward: -20.500000, loss: 0.001308, epsilon: 0.094019, episode:   79\n",
      "frames: 75000, reward: -20.500000, loss: 0.001025, epsilon: 0.091264, episode:   80\n",
      "frames: 76000, reward: -20.600000, loss: 0.002388, epsilon: 0.088600, episode:   82\n",
      "frames: 77000, reward: -20.600000, loss: 0.002742, epsilon: 0.086023, episode:   82\n",
      "frames: 78000, reward: -20.300000, loss: 0.006721, epsilon: 0.083531, episode:   83\n",
      "frames: 79000, reward: -20.400000, loss: 0.000944, epsilon: 0.081120, episode:   84\n",
      "frames: 80000, reward: -20.500000, loss: 0.001839, epsilon: 0.078789, episode:   85\n",
      "frames: 81000, reward: -20.200000, loss: 0.004038, epsilon: 0.076533, episode:   86\n",
      "frames: 82000, reward: -20.200000, loss: 0.002567, epsilon: 0.074352, episode:   87\n",
      "frames: 83000, reward: -20.100000, loss: 0.003744, epsilon: 0.072243, episode:   88\n",
      "frames: 84000, reward: -20.200000, loss: 0.001805, epsilon: 0.070202, episode:   89\n",
      "frames: 85000, reward: -20.200000, loss: 0.000923, epsilon: 0.068228, episode:   90\n",
      "frames: 86000, reward: -20.200000, loss: 0.016099, epsilon: 0.066319, episode:   91\n",
      "frames: 87000, reward: -20.500000, loss: 0.004695, epsilon: 0.064473, episode:   93\n",
      "frames: 88000, reward: -20.500000, loss: 0.004109, epsilon: 0.062687, episode:   93\n",
      "frames: 89000, reward: -20.500000, loss: 0.001396, epsilon: 0.060960, episode:   94\n",
      "frames: 90000, reward: -20.400000, loss: 0.003285, epsilon: 0.059289, episode:   95\n",
      "frames: 91000, reward: -20.700000, loss: 0.001124, epsilon: 0.057673, episode:   96\n",
      "frames: 92000, reward: -20.700000, loss: 0.002942, epsilon: 0.056110, episode:   97\n",
      "frames: 93000, reward: -20.400000, loss: 0.002049, epsilon: 0.054599, episode:   98\n",
      "frames: 94000, reward: -20.400000, loss: 0.002142, epsilon: 0.053137, episode:   99\n",
      "frames: 95000, reward: -20.400000, loss: 0.002608, epsilon: 0.051722, episode:   99\n",
      "frames: 96000, reward: -20.300000, loss: 0.002862, epsilon: 0.050355, episode:  100\n",
      "frames: 97000, reward: -19.700000, loss: 0.002531, epsilon: 0.049032, episode:  101\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 98000, reward: -19.600000, loss: 0.015657, epsilon: 0.047752, episode:  102\n",
      "frames: 99000, reward: -19.400000, loss: 0.001237, epsilon: 0.046514, episode:  103\n",
      "frames: 100000, reward: -19.400000, loss: 0.001553, epsilon: 0.045317, episode:  103\n",
      "frames: 101000, reward: -19.100000, loss: 0.001564, epsilon: 0.044159, episode:  104\n",
      "frames: 102000, reward: -18.700000, loss: 0.008827, epsilon: 0.043040, episode:  105\n",
      "frames: 103000, reward: -18.500000, loss: 0.027877, epsilon: 0.041956, episode:  106\n",
      "frames: 104000, reward: -18.500000, loss: 0.006416, epsilon: 0.040909, episode:  107\n",
      "frames: 105000, reward: -18.500000, loss: 0.001062, epsilon: 0.039895, episode:  107\n",
      "frames: 106000, reward: -18.900000, loss: 0.001452, epsilon: 0.038915, episode:  108\n",
      "frames: 107000, reward: -18.700000, loss: 0.002001, epsilon: 0.037967, episode:  109\n",
      "frames: 108000, reward: -18.700000, loss: 0.001621, epsilon: 0.037050, episode:  109\n",
      "frames: 109000, reward: -18.400000, loss: 0.001218, epsilon: 0.036164, episode:  110\n",
      "frames: 110000, reward: -18.900000, loss: 0.000782, epsilon: 0.035306, episode:  111\n",
      "frames: 111000, reward: -18.900000, loss: 0.001047, epsilon: 0.034476, episode:  111\n",
      "frames: 112000, reward: -18.400000, loss: 0.005930, epsilon: 0.033674, episode:  112\n",
      "frames: 113000, reward: -18.400000, loss: 0.001568, epsilon: 0.032898, episode:  112\n",
      "frames: 114000, reward: -18.500000, loss: 0.003673, epsilon: 0.032147, episode:  113\n",
      "frames: 115000, reward: -18.700000, loss: 0.001006, epsilon: 0.031421, episode:  114\n",
      "frames: 116000, reward: -19.000000, loss: 0.002280, epsilon: 0.030719, episode:  115\n",
      "frames: 117000, reward: -19.100000, loss: 0.010821, epsilon: 0.030039, episode:  116\n",
      "frames: 118000, reward: -19.000000, loss: 0.003580, epsilon: 0.029383, episode:  117\n",
      "frames: 119000, reward: -19.000000, loss: 0.002835, epsilon: 0.028747, episode:  117\n",
      "frames: 120000, reward: -18.800000, loss: 0.005160, epsilon: 0.028132, episode:  118\n",
      "frames: 121000, reward: -18.800000, loss: 0.001540, epsilon: 0.027538, episode:  119\n",
      "frames: 122000, reward: -19.100000, loss: 0.001590, epsilon: 0.026963, episode:  120\n",
      "frames: 123000, reward: -19.100000, loss: 0.001970, epsilon: 0.026407, episode:  120\n",
      "frames: 124000, reward: -19.000000, loss: 0.009561, epsilon: 0.025869, episode:  121\n",
      "frames: 125000, reward: -19.000000, loss: 0.012405, epsilon: 0.025349, episode:  121\n",
      "frames: 126000, reward: -19.200000, loss: 0.000970, epsilon: 0.024846, episode:  122\n",
      "frames: 127000, reward: -19.200000, loss: 0.001555, epsilon: 0.024359, episode:  122\n",
      "frames: 128000, reward: -18.600000, loss: 0.000881, epsilon: 0.023888, episode:  123\n",
      "frames: 129000, reward: -18.600000, loss: 0.002415, epsilon: 0.023433, episode:  123\n",
      "frames: 130000, reward: -18.200000, loss: 0.007677, epsilon: 0.022992, episode:  124\n",
      "frames: 131000, reward: -18.000000, loss: 0.017362, epsilon: 0.022567, episode:  125\n",
      "frames: 132000, reward: -18.000000, loss: 0.004701, epsilon: 0.022155, episode:  125\n",
      "frames: 133000, reward: -17.100000, loss: 0.003436, epsilon: 0.021756, episode:  126\n",
      "frames: 134000, reward: -17.100000, loss: 0.002151, epsilon: 0.021371, episode:  126\n",
      "frames: 135000, reward: -16.600000, loss: 0.001394, epsilon: 0.020998, episode:  127\n",
      "frames: 136000, reward: -16.600000, loss: 0.001580, epsilon: 0.020637, episode:  127\n",
      "frames: 137000, reward: -16.600000, loss: 0.003193, epsilon: 0.020289, episode:  127\n",
      "frames: 138000, reward: -15.700000, loss: 0.004688, epsilon: 0.019951, episode:  128\n",
      "frames: 139000, reward: -15.700000, loss: 0.007328, epsilon: 0.019625, episode:  128\n",
      "frames: 140000, reward: -15.600000, loss: 0.001452, epsilon: 0.019310, episode:  129\n",
      "frames: 141000, reward: -15.600000, loss: 0.002281, epsilon: 0.019004, episode:  129\n",
      "frames: 142000, reward: -15.100000, loss: 0.002284, epsilon: 0.018709, episode:  130\n",
      "frames: 143000, reward: -15.100000, loss: 0.002226, epsilon: 0.018424, episode:  130\n",
      "frames: 144000, reward: -14.900000, loss: 0.001948, epsilon: 0.018147, episode:  131\n",
      "frames: 145000, reward: -14.900000, loss: 0.002260, epsilon: 0.017880, episode:  131\n",
      "frames: 146000, reward: -14.800000, loss: 0.003583, epsilon: 0.017622, episode:  132\n",
      "frames: 147000, reward: -15.500000, loss: 0.002880, epsilon: 0.017372, episode:  133\n",
      "frames: 148000, reward: -15.500000, loss: 0.001021, epsilon: 0.017130, episode:  133\n",
      "frames: 149000, reward: -15.600000, loss: 0.001017, epsilon: 0.016897, episode:  134\n",
      "frames: 150000, reward: -15.800000, loss: 0.002704, epsilon: 0.016671, episode:  135\n",
      "frames: 151000, reward: -15.800000, loss: 0.001674, epsilon: 0.016452, episode:  135\n",
      "frames: 152000, reward: -16.400000, loss: 0.001504, epsilon: 0.016240, episode:  136\n",
      "frames: 153000, reward: -16.400000, loss: 0.003816, epsilon: 0.016036, episode:  136\n",
      "frames: 154000, reward: -16.500000, loss: 0.001872, epsilon: 0.015838, episode:  137\n",
      "frames: 155000, reward: -16.500000, loss: 0.003604, epsilon: 0.015647, episode:  137\n",
      "frames: 156000, reward: -17.200000, loss: 0.001973, epsilon: 0.015461, episode:  138\n",
      "frames: 157000, reward: -17.200000, loss: 0.002601, epsilon: 0.015282, episode:  138\n",
      "frames: 158000, reward: -17.200000, loss: 0.006655, epsilon: 0.015109, episode:  138\n",
      "frames: 159000, reward: -16.700000, loss: 0.002071, epsilon: 0.014942, episode:  139\n",
      "frames: 160000, reward: -16.700000, loss: 0.002343, epsilon: 0.014780, episode:  139\n",
      "frames: 161000, reward: -17.100000, loss: 0.005260, epsilon: 0.014623, episode:  140\n",
      "frames: 162000, reward: -17.100000, loss: 0.003200, epsilon: 0.014471, episode:  140\n",
      "frames: 163000, reward: -17.100000, loss: 0.000678, epsilon: 0.014325, episode:  140\n",
      "frames: 164000, reward: -17.100000, loss: 0.001514, epsilon: 0.014183, episode:  141\n",
      "frames: 165000, reward: -17.100000, loss: 0.003181, epsilon: 0.014046, episode:  141\n",
      "frames: 166000, reward: -17.200000, loss: 0.003889, epsilon: 0.013913, episode:  142\n",
      "frames: 167000, reward: -17.200000, loss: 0.002344, epsilon: 0.013785, episode:  142\n",
      "frames: 168000, reward: -17.200000, loss: 0.001479, epsilon: 0.013661, episode:  142\n",
      "frames: 169000, reward: -16.400000, loss: 0.001195, epsilon: 0.013541, episode:  143\n",
      "frames: 170000, reward: -16.400000, loss: 0.002614, epsilon: 0.013425, episode:  143\n",
      "frames: 171000, reward: -16.300000, loss: 0.002619, epsilon: 0.013313, episode:  144\n",
      "frames: 172000, reward: -16.300000, loss: 0.001349, epsilon: 0.013204, episode:  144\n",
      "frames: 173000, reward: -15.900000, loss: 0.002030, epsilon: 0.013099, episode:  145\n",
      "frames: 174000, reward: -15.900000, loss: 0.001980, epsilon: 0.012997, episode:  145\n",
      "frames: 175000, reward: -15.900000, loss: 0.002244, epsilon: 0.012899, episode:  145\n",
      "frames: 176000, reward: -15.500000, loss: 0.002287, epsilon: 0.012804, episode:  146\n",
      "frames: 177000, reward: -15.500000, loss: 0.000987, epsilon: 0.012712, episode:  146\n",
      "frames: 178000, reward: -15.800000, loss: 0.001368, epsilon: 0.012623, episode:  147\n",
      "frames: 179000, reward: -15.800000, loss: 0.003011, epsilon: 0.012537, episode:  147\n",
      "frames: 180000, reward: -15.800000, loss: 0.001444, epsilon: 0.012454, episode:  147\n",
      "frames: 181000, reward: -15.300000, loss: 0.001995, epsilon: 0.012374, episode:  148\n",
      "frames: 182000, reward: -15.300000, loss: 0.002753, epsilon: 0.012296, episode:  148\n",
      "frames: 183000, reward: -15.800000, loss: 0.001488, epsilon: 0.012220, episode:  149\n",
      "frames: 184000, reward: -15.800000, loss: 0.002250, epsilon: 0.012148, episode:  149\n",
      "frames: 185000, reward: -15.900000, loss: 0.003603, epsilon: 0.012077, episode:  150\n",
      "frames: 186000, reward: -15.900000, loss: 0.001461, epsilon: 0.012009, episode:  150\n",
      "frames: 187000, reward: -15.900000, loss: 0.002716, epsilon: 0.011943, episode:  150\n",
      "frames: 188000, reward: -15.500000, loss: 0.000953, epsilon: 0.011880, episode:  151\n",
      "frames: 189000, reward: -15.500000, loss: 0.001051, epsilon: 0.011818, episode:  151\n",
      "frames: 190000, reward: -15.500000, loss: 0.000524, epsilon: 0.011758, episode:  151\n",
      "frames: 191000, reward: -15.000000, loss: 0.001229, epsilon: 0.011701, episode:  152\n",
      "frames: 192000, reward: -15.000000, loss: 0.001686, epsilon: 0.011645, episode:  152\n",
      "frames: 193000, reward: -15.000000, loss: 0.000922, epsilon: 0.011591, episode:  152\n",
      "frames: 194000, reward: -14.800000, loss: 0.002499, epsilon: 0.011539, episode:  153\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 195000, reward: -14.800000, loss: 0.001819, epsilon: 0.011488, episode:  153\n",
      "frames: 196000, reward: -14.800000, loss: 0.001078, epsilon: 0.011440, episode:  153\n",
      "frames: 197000, reward: -14.200000, loss: 0.003934, epsilon: 0.011392, episode:  154\n",
      "frames: 198000, reward: -14.300000, loss: 0.002505, epsilon: 0.011347, episode:  155\n",
      "frames: 199000, reward: -14.300000, loss: 0.002658, epsilon: 0.011303, episode:  155\n",
      "frames: 200000, reward: -14.500000, loss: 0.001108, epsilon: 0.011260, episode:  156\n",
      "frames: 201000, reward: -14.500000, loss: 0.002462, epsilon: 0.011219, episode:  157\n",
      "frames: 202000, reward: -14.500000, loss: 0.000494, epsilon: 0.011179, episode:  157\n",
      "frames: 203000, reward: -14.500000, loss: 0.001962, epsilon: 0.011140, episode:  157\n",
      "frames: 204000, reward: -14.900000, loss: 0.001358, epsilon: 0.011103, episode:  158\n",
      "frames: 205000, reward: -14.900000, loss: 0.001843, epsilon: 0.011066, episode:  158\n",
      "frames: 206000, reward: -14.300000, loss: 0.001423, epsilon: 0.011032, episode:  159\n",
      "frames: 207000, reward: -14.300000, loss: 0.002135, epsilon: 0.010998, episode:  159\n",
      "frames: 208000, reward: -14.300000, loss: 0.000616, epsilon: 0.010965, episode:  159\n",
      "frames: 209000, reward: -13.900000, loss: 0.002860, epsilon: 0.010933, episode:  160\n",
      "frames: 210000, reward: -13.900000, loss: 0.001357, epsilon: 0.010903, episode:  160\n",
      "frames: 211000, reward: -14.100000, loss: 0.001572, epsilon: 0.010873, episode:  161\n",
      "frames: 212000, reward: -14.100000, loss: 0.001406, epsilon: 0.010845, episode:  161\n",
      "frames: 213000, reward: -14.100000, loss: 0.002769, epsilon: 0.010817, episode:  161\n",
      "frames: 214000, reward: -14.400000, loss: 0.001749, epsilon: 0.010790, episode:  162\n",
      "frames: 215000, reward: -14.400000, loss: 0.000988, epsilon: 0.010764, episode:  162\n",
      "frames: 216000, reward: -14.400000, loss: 0.001718, epsilon: 0.010739, episode:  162\n",
      "frames: 217000, reward: -15.200000, loss: 0.001679, epsilon: 0.010715, episode:  163\n",
      "frames: 218000, reward: -15.200000, loss: 0.004270, epsilon: 0.010691, episode:  163\n",
      "frames: 219000, reward: -15.200000, loss: 0.000943, epsilon: 0.010669, episode:  163\n",
      "frames: 220000, reward: -15.700000, loss: 0.002501, epsilon: 0.010647, episode:  164\n",
      "frames: 221000, reward: -15.700000, loss: 0.002576, epsilon: 0.010626, episode:  164\n",
      "frames: 222000, reward: -16.000000, loss: 0.001681, epsilon: 0.010605, episode:  165\n",
      "frames: 223000, reward: -16.000000, loss: 0.002584, epsilon: 0.010585, episode:  165\n",
      "frames: 224000, reward: -16.300000, loss: 0.002985, epsilon: 0.010566, episode:  166\n",
      "frames: 225000, reward: -16.300000, loss: 0.002583, epsilon: 0.010548, episode:  166\n",
      "frames: 226000, reward: -16.300000, loss: 0.000761, epsilon: 0.010530, episode:  166\n",
      "frames: 227000, reward: -16.300000, loss: 0.001893, epsilon: 0.010512, episode:  166\n",
      "frames: 228000, reward: -15.500000, loss: 0.003981, epsilon: 0.010495, episode:  167\n",
      "frames: 229000, reward: -15.500000, loss: 0.001527, epsilon: 0.010479, episode:  167\n",
      "frames: 230000, reward: -15.500000, loss: 0.001673, epsilon: 0.010463, episode:  167\n",
      "frames: 231000, reward: -15.300000, loss: 0.001196, epsilon: 0.010448, episode:  168\n",
      "frames: 232000, reward: -15.300000, loss: 0.002633, epsilon: 0.010434, episode:  168\n",
      "frames: 233000, reward: -15.300000, loss: 0.001972, epsilon: 0.010419, episode:  168\n",
      "frames: 234000, reward: -15.800000, loss: 0.001683, epsilon: 0.010406, episode:  169\n",
      "frames: 235000, reward: -15.800000, loss: 0.001234, epsilon: 0.010392, episode:  169\n",
      "frames: 236000, reward: -15.800000, loss: 0.001755, epsilon: 0.010379, episode:  169\n",
      "frames: 237000, reward: -15.700000, loss: 0.001569, epsilon: 0.010367, episode:  170\n",
      "frames: 238000, reward: -15.700000, loss: 0.000804, epsilon: 0.010355, episode:  170\n",
      "frames: 239000, reward: -15.700000, loss: 0.001233, epsilon: 0.010343, episode:  170\n",
      "frames: 240000, reward: -15.600000, loss: 0.002289, epsilon: 0.010332, episode:  171\n",
      "frames: 241000, reward: -15.600000, loss: 0.000735, epsilon: 0.010321, episode:  171\n",
      "frames: 242000, reward: -15.600000, loss: 0.001162, epsilon: 0.010311, episode:  171\n",
      "frames: 243000, reward: -15.600000, loss: 0.001445, epsilon: 0.010301, episode:  171\n",
      "frames: 244000, reward: -15.400000, loss: 0.000946, epsilon: 0.010291, episode:  172\n",
      "frames: 245000, reward: -15.400000, loss: 0.001107, epsilon: 0.010281, episode:  172\n",
      "frames: 246000, reward: -15.400000, loss: 0.001182, epsilon: 0.010272, episode:  172\n",
      "frames: 247000, reward: -14.500000, loss: 0.002069, epsilon: 0.010263, episode:  173\n",
      "frames: 248000, reward: -14.500000, loss: 0.000535, epsilon: 0.010254, episode:  173\n",
      "frames: 249000, reward: -14.500000, loss: 0.002218, epsilon: 0.010246, episode:  173\n",
      "frames: 250000, reward: -14.500000, loss: 0.003110, epsilon: 0.010238, episode:  173\n",
      "frames: 251000, reward: -14.500000, loss: 0.001040, epsilon: 0.010230, episode:  173\n",
      "frames: 252000, reward: -13.600000, loss: 0.000788, epsilon: 0.010223, episode:  174\n",
      "frames: 253000, reward: -13.600000, loss: 0.002012, epsilon: 0.010215, episode:  174\n",
      "frames: 254000, reward: -13.600000, loss: 0.001104, epsilon: 0.010208, episode:  174\n",
      "frames: 255000, reward: -13.600000, loss: 0.002746, epsilon: 0.010201, episode:  174\n",
      "frames: 256000, reward: -12.300000, loss: 0.001152, epsilon: 0.010195, episode:  175\n",
      "frames: 257000, reward: -12.300000, loss: 0.002381, epsilon: 0.010188, episode:  175\n",
      "frames: 258000, reward: -12.300000, loss: 0.001137, epsilon: 0.010182, episode:  175\n",
      "frames: 259000, reward: -12.300000, loss: 0.001411, epsilon: 0.010176, episode:  175\n",
      "frames: 260000, reward: -11.400000, loss: 0.001843, epsilon: 0.010171, episode:  176\n",
      "frames: 261000, reward: -11.400000, loss: 0.000986, epsilon: 0.010165, episode:  176\n",
      "frames: 262000, reward: -11.400000, loss: 0.001822, epsilon: 0.010160, episode:  176\n",
      "frames: 263000, reward: -11.800000, loss: 0.001870, epsilon: 0.010154, episode:  177\n",
      "frames: 264000, reward: -11.800000, loss: 0.001520, epsilon: 0.010149, episode:  177\n",
      "frames: 265000, reward: -11.800000, loss: 0.000923, epsilon: 0.010144, episode:  177\n",
      "frames: 266000, reward: -11.800000, loss: 0.003116, epsilon: 0.010140, episode:  177\n",
      "frames: 267000, reward: -11.300000, loss: 0.000878, epsilon: 0.010135, episode:  178\n",
      "frames: 268000, reward: -11.300000, loss: 0.001066, epsilon: 0.010131, episode:  178\n",
      "frames: 269000, reward: -11.300000, loss: 0.002233, epsilon: 0.010126, episode:  178\n",
      "frames: 270000, reward: -11.300000, loss: 0.000729, epsilon: 0.010122, episode:  178\n",
      "frames: 271000, reward: -11.300000, loss: 0.001726, epsilon: 0.010118, episode:  178\n",
      "frames: 272000, reward: -10.000000, loss: 0.001319, epsilon: 0.010114, episode:  179\n",
      "frames: 273000, reward: -10.000000, loss: 0.000985, epsilon: 0.010111, episode:  179\n",
      "frames: 274000, reward: -10.000000, loss: 0.000840, epsilon: 0.010107, episode:  179\n",
      "frames: 275000, reward: -10.100000, loss: 0.021015, epsilon: 0.010103, episode:  180\n",
      "frames: 276000, reward: -10.100000, loss: 0.001744, epsilon: 0.010100, episode:  180\n",
      "frames: 277000, reward: -10.100000, loss: 0.002037, epsilon: 0.010097, episode:  180\n",
      "frames: 278000, reward: -10.100000, loss: 0.003581, epsilon: 0.010094, episode:  180\n",
      "frames: 279000, reward: -9.500000, loss: 0.000702, epsilon: 0.010091, episode:  181\n",
      "frames: 280000, reward: -9.500000, loss: 0.001948, epsilon: 0.010088, episode:  181\n",
      "frames: 281000, reward: -9.500000, loss: 0.001526, epsilon: 0.010085, episode:  181\n",
      "frames: 282000, reward: -9.500000, loss: 0.000478, epsilon: 0.010082, episode:  181\n",
      "frames: 283000, reward: -9.300000, loss: 0.002354, epsilon: 0.010079, episode:  182\n",
      "frames: 284000, reward: -9.300000, loss: 0.001905, epsilon: 0.010077, episode:  182\n",
      "frames: 285000, reward: -9.300000, loss: 0.000904, epsilon: 0.010074, episode:  182\n",
      "frames: 286000, reward: -9.300000, loss: 0.000755, epsilon: 0.010072, episode:  182\n",
      "frames: 287000, reward: -9.300000, loss: 0.000728, epsilon: 0.010069, episode:  182\n",
      "frames: 288000, reward: -8.700000, loss: 0.000848, epsilon: 0.010067, episode:  183\n",
      "frames: 289000, reward: -8.700000, loss: 0.001336, epsilon: 0.010065, episode:  183\n",
      "frames: 290000, reward: -8.700000, loss: 0.000829, epsilon: 0.010063, episode:  183\n",
      "frames: 291000, reward: -8.700000, loss: 0.000761, epsilon: 0.010061, episode:  183\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 292000, reward: -8.600000, loss: 0.001046, epsilon: 0.010059, episode:  184\n",
      "frames: 293000, reward: -8.600000, loss: 0.001206, epsilon: 0.010057, episode:  184\n",
      "frames: 294000, reward: -8.600000, loss: 0.002859, epsilon: 0.010055, episode:  184\n",
      "frames: 295000, reward: -8.600000, loss: 0.001287, epsilon: 0.010053, episode:  184\n",
      "frames: 296000, reward: -9.100000, loss: 0.003105, epsilon: 0.010051, episode:  185\n",
      "frames: 297000, reward: -9.100000, loss: 0.001499, epsilon: 0.010050, episode:  185\n",
      "frames: 298000, reward: -9.100000, loss: 0.000990, epsilon: 0.010048, episode:  185\n",
      "frames: 299000, reward: -9.100000, loss: 0.001985, epsilon: 0.010046, episode:  185\n",
      "frames: 300000, reward: -9.100000, loss: 0.001109, epsilon: 0.010045, episode:  185\n",
      "frames: 301000, reward: -9.100000, loss: 0.001570, epsilon: 0.010043, episode:  186\n",
      "frames: 302000, reward: -9.100000, loss: 0.002088, epsilon: 0.010042, episode:  186\n",
      "frames: 303000, reward: -9.100000, loss: 0.001006, epsilon: 0.010041, episode:  186\n",
      "frames: 304000, reward: -9.100000, loss: 0.001273, epsilon: 0.010039, episode:  187\n",
      "frames: 305000, reward: -9.100000, loss: 0.000646, epsilon: 0.010038, episode:  187\n",
      "frames: 306000, reward: -9.100000, loss: 0.000883, epsilon: 0.010037, episode:  187\n",
      "frames: 307000, reward: -9.400000, loss: 0.000963, epsilon: 0.010036, episode:  188\n",
      "frames: 308000, reward: -9.400000, loss: 0.001539, epsilon: 0.010034, episode:  188\n",
      "frames: 309000, reward: -9.400000, loss: 0.001693, epsilon: 0.010033, episode:  188\n",
      "frames: 310000, reward: -9.400000, loss: 0.002032, epsilon: 0.010032, episode:  188\n",
      "frames: 311000, reward: -9.800000, loss: 0.000770, epsilon: 0.010031, episode:  189\n",
      "frames: 312000, reward: -9.800000, loss: 0.001683, epsilon: 0.010030, episode:  189\n",
      "frames: 313000, reward: -9.800000, loss: 0.001368, epsilon: 0.010029, episode:  189\n",
      "frames: 314000, reward: -9.800000, loss: 0.000680, epsilon: 0.010028, episode:  189\n",
      "frames: 315000, reward: -8.800000, loss: 0.002007, epsilon: 0.010027, episode:  190\n",
      "frames: 316000, reward: -8.800000, loss: 0.001385, epsilon: 0.010026, episode:  190\n",
      "frames: 317000, reward: -8.800000, loss: 0.001124, epsilon: 0.010026, episode:  190\n",
      "frames: 318000, reward: -9.300000, loss: 0.001075, epsilon: 0.010025, episode:  191\n",
      "frames: 319000, reward: -9.300000, loss: 0.001390, epsilon: 0.010024, episode:  191\n",
      "frames: 320000, reward: -9.300000, loss: 0.001098, epsilon: 0.010023, episode:  191\n",
      "frames: 321000, reward: -9.300000, loss: 0.001819, epsilon: 0.010022, episode:  191\n",
      "frames: 322000, reward: -9.200000, loss: 0.001416, epsilon: 0.010022, episode:  192\n",
      "frames: 323000, reward: -9.200000, loss: 0.000992, epsilon: 0.010021, episode:  192\n",
      "frames: 324000, reward: -9.200000, loss: 0.000644, epsilon: 0.010020, episode:  192\n",
      "frames: 325000, reward: -9.200000, loss: 0.001304, epsilon: 0.010020, episode:  192\n",
      "frames: 326000, reward: -9.200000, loss: 0.001602, epsilon: 0.010019, episode:  192\n",
      "frames: 327000, reward: -9.800000, loss: 0.001487, epsilon: 0.010018, episode:  193\n",
      "frames: 328000, reward: -9.800000, loss: 0.001049, epsilon: 0.010018, episode:  193\n",
      "frames: 329000, reward: -9.800000, loss: 0.002534, epsilon: 0.010017, episode:  193\n",
      "frames: 330000, reward: -10.300000, loss: 0.001238, epsilon: 0.010017, episode:  194\n",
      "frames: 331000, reward: -10.300000, loss: 0.001450, epsilon: 0.010016, episode:  194\n",
      "frames: 332000, reward: -10.300000, loss: 0.006630, epsilon: 0.010015, episode:  194\n",
      "frames: 333000, reward: -10.300000, loss: 0.002344, epsilon: 0.010015, episode:  194\n",
      "frames: 334000, reward: -10.100000, loss: 0.001837, epsilon: 0.010014, episode:  195\n",
      "frames: 335000, reward: -10.100000, loss: 0.001318, epsilon: 0.010014, episode:  195\n",
      "frames: 336000, reward: -10.100000, loss: 0.001047, epsilon: 0.010014, episode:  195\n",
      "frames: 337000, reward: -10.100000, loss: 0.002349, epsilon: 0.010013, episode:  195\n",
      "frames: 338000, reward: -9.600000, loss: 0.000783, epsilon: 0.010013, episode:  196\n",
      "frames: 339000, reward: -9.600000, loss: 0.001562, epsilon: 0.010012, episode:  196\n",
      "frames: 340000, reward: -9.600000, loss: 0.000834, epsilon: 0.010012, episode:  196\n",
      "frames: 341000, reward: -9.600000, loss: 0.002444, epsilon: 0.010011, episode:  196\n",
      "frames: 342000, reward: -9.600000, loss: 0.001422, epsilon: 0.010011, episode:  196\n",
      "frames: 343000, reward: -9.600000, loss: 0.001161, epsilon: 0.010011, episode:  196\n",
      "frames: 344000, reward: -9.600000, loss: 0.001164, epsilon: 0.010010, episode:  196\n",
      "frames: 345000, reward: -8.300000, loss: 0.000772, epsilon: 0.010010, episode:  197\n",
      "frames: 346000, reward: -8.300000, loss: 0.001777, epsilon: 0.010010, episode:  197\n",
      "frames: 347000, reward: -8.800000, loss: 0.001010, epsilon: 0.010009, episode:  198\n",
      "frames: 348000, reward: -8.800000, loss: 0.001344, epsilon: 0.010009, episode:  198\n",
      "frames: 349000, reward: -8.800000, loss: 0.000862, epsilon: 0.010009, episode:  198\n",
      "frames: 350000, reward: -9.200000, loss: 0.002538, epsilon: 0.010008, episode:  199\n",
      "frames: 351000, reward: -9.200000, loss: 0.002063, epsilon: 0.010008, episode:  199\n",
      "frames: 352000, reward: -9.200000, loss: 0.000453, epsilon: 0.010008, episode:  199\n",
      "frames: 353000, reward: -9.200000, loss: 0.000839, epsilon: 0.010008, episode:  199\n",
      "frames: 354000, reward: -9.700000, loss: 0.002502, epsilon: 0.010007, episode:  200\n",
      "frames: 355000, reward: -9.700000, loss: 0.001923, epsilon: 0.010007, episode:  200\n",
      "frames: 356000, reward: -9.700000, loss: 0.000776, epsilon: 0.010007, episode:  200\n",
      "frames: 357000, reward: -9.700000, loss: 0.002065, epsilon: 0.010007, episode:  200\n",
      "frames: 358000, reward: -9.700000, loss: 0.001412, epsilon: 0.010007, episode:  200\n",
      "frames: 359000, reward: -7.700000, loss: 0.000814, epsilon: 0.010006, episode:  201\n",
      "frames: 360000, reward: -7.700000, loss: 0.000616, epsilon: 0.010006, episode:  201\n",
      "frames: 361000, reward: -7.700000, loss: 0.001107, epsilon: 0.010006, episode:  201\n",
      "frames: 362000, reward: -7.700000, loss: 0.001812, epsilon: 0.010006, episode:  202\n",
      "frames: 363000, reward: -7.700000, loss: 0.001152, epsilon: 0.010006, episode:  202\n",
      "frames: 364000, reward: -7.700000, loss: 0.000597, epsilon: 0.010005, episode:  202\n",
      "frames: 365000, reward: -7.700000, loss: 0.001332, epsilon: 0.010005, episode:  202\n",
      "frames: 366000, reward: -7.700000, loss: 0.000636, epsilon: 0.010005, episode:  202\n",
      "frames: 367000, reward: -7.300000, loss: 0.000304, epsilon: 0.010005, episode:  203\n",
      "frames: 368000, reward: -7.300000, loss: 0.000708, epsilon: 0.010005, episode:  203\n",
      "frames: 369000, reward: -7.300000, loss: 0.000856, epsilon: 0.010005, episode:  203\n",
      "frames: 370000, reward: -7.300000, loss: 0.000984, epsilon: 0.010004, episode:  203\n",
      "frames: 371000, reward: -6.400000, loss: 0.001253, epsilon: 0.010004, episode:  204\n",
      "frames: 372000, reward: -6.400000, loss: 0.001880, epsilon: 0.010004, episode:  204\n",
      "frames: 373000, reward: -6.400000, loss: 0.000729, epsilon: 0.010004, episode:  204\n",
      "frames: 374000, reward: -6.400000, loss: 0.001323, epsilon: 0.010004, episode:  204\n",
      "frames: 375000, reward: -6.300000, loss: 0.000814, epsilon: 0.010004, episode:  205\n",
      "frames: 376000, reward: -6.300000, loss: 0.001585, epsilon: 0.010004, episode:  205\n",
      "frames: 377000, reward: -6.300000, loss: 0.001329, epsilon: 0.010003, episode:  205\n",
      "frames: 378000, reward: -6.300000, loss: 0.000898, epsilon: 0.010003, episode:  205\n",
      "frames: 379000, reward: -5.300000, loss: 0.000940, epsilon: 0.010003, episode:  206\n",
      "frames: 380000, reward: -5.300000, loss: 0.000484, epsilon: 0.010003, episode:  206\n",
      "frames: 381000, reward: -5.300000, loss: 0.000621, epsilon: 0.010003, episode:  206\n",
      "frames: 382000, reward: -5.300000, loss: 0.001891, epsilon: 0.010003, episode:  206\n",
      "frames: 383000, reward: -5.400000, loss: 0.000321, epsilon: 0.010003, episode:  207\n",
      "frames: 384000, reward: -5.400000, loss: 0.001119, epsilon: 0.010003, episode:  207\n",
      "frames: 385000, reward: -5.400000, loss: 0.001911, epsilon: 0.010003, episode:  207\n",
      "frames: 386000, reward: -5.400000, loss: 0.001592, epsilon: 0.010003, episode:  207\n",
      "frames: 387000, reward: -5.400000, loss: 0.001156, epsilon: 0.010002, episode:  207\n",
      "frames: 388000, reward: -4.200000, loss: 0.006552, epsilon: 0.010002, episode:  208\n",
      "frames: 389000, reward: -4.200000, loss: 0.001592, epsilon: 0.010002, episode:  208\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 390000, reward: -4.200000, loss: 0.001531, epsilon: 0.010002, episode:  208\n",
      "frames: 391000, reward: -4.200000, loss: 0.000346, epsilon: 0.010002, episode:  208\n",
      "frames: 392000, reward: -3.700000, loss: 0.002486, epsilon: 0.010002, episode:  209\n",
      "frames: 393000, reward: -3.700000, loss: 0.001885, epsilon: 0.010002, episode:  209\n",
      "frames: 394000, reward: -3.700000, loss: 0.000900, epsilon: 0.010002, episode:  209\n",
      "frames: 395000, reward: -3.700000, loss: 0.001096, epsilon: 0.010002, episode:  209\n",
      "frames: 396000, reward: -4.200000, loss: 0.001590, epsilon: 0.010002, episode:  210\n",
      "frames: 397000, reward: -4.200000, loss: 0.001950, epsilon: 0.010002, episode:  210\n",
      "frames: 398000, reward: -4.200000, loss: 0.002162, epsilon: 0.010002, episode:  210\n",
      "frames: 399000, reward: -5.600000, loss: 0.001480, epsilon: 0.010002, episode:  211\n",
      "frames: 400000, reward: -5.600000, loss: 0.000844, epsilon: 0.010002, episode:  211\n",
      "frames: 401000, reward: -5.600000, loss: 0.000613, epsilon: 0.010002, episode:  211\n",
      "frames: 402000, reward: -5.600000, loss: 0.000413, epsilon: 0.010001, episode:  211\n",
      "frames: 403000, reward: -4.800000, loss: 0.002316, epsilon: 0.010001, episode:  212\n",
      "frames: 404000, reward: -4.800000, loss: 0.001423, epsilon: 0.010001, episode:  212\n",
      "frames: 405000, reward: -4.800000, loss: 0.000297, epsilon: 0.010001, episode:  212\n",
      "frames: 406000, reward: -4.800000, loss: 0.000461, epsilon: 0.010001, episode:  212\n",
      "frames: 407000, reward: -4.600000, loss: 0.001232, epsilon: 0.010001, episode:  213\n",
      "frames: 408000, reward: -4.600000, loss: 0.000789, epsilon: 0.010001, episode:  213\n",
      "frames: 409000, reward: -4.600000, loss: 0.003560, epsilon: 0.010001, episode:  213\n",
      "frames: 410000, reward: -4.600000, loss: 0.000852, epsilon: 0.010001, episode:  213\n",
      "frames: 411000, reward: -4.600000, loss: 0.001211, epsilon: 0.010001, episode:  213\n",
      "frames: 412000, reward: -4.100000, loss: 0.000844, epsilon: 0.010001, episode:  214\n",
      "frames: 413000, reward: -4.100000, loss: 0.000478, epsilon: 0.010001, episode:  214\n",
      "frames: 414000, reward: -4.100000, loss: 0.001646, epsilon: 0.010001, episode:  214\n",
      "frames: 415000, reward: -4.100000, loss: 0.000614, epsilon: 0.010001, episode:  214\n",
      "frames: 416000, reward: -4.100000, loss: 0.000992, epsilon: 0.010001, episode:  214\n",
      "frames: 417000, reward: -3.200000, loss: 0.000579, epsilon: 0.010001, episode:  215\n",
      "frames: 418000, reward: -3.200000, loss: 0.000996, epsilon: 0.010001, episode:  215\n",
      "frames: 419000, reward: -3.200000, loss: 0.000344, epsilon: 0.010001, episode:  215\n",
      "frames: 420000, reward: -3.200000, loss: 0.001502, epsilon: 0.010001, episode:  215\n",
      "frames: 421000, reward: -4.500000, loss: 0.003024, epsilon: 0.010001, episode:  216\n",
      "frames: 422000, reward: -4.500000, loss: 0.000644, epsilon: 0.010001, episode:  216\n",
      "frames: 423000, reward: -4.500000, loss: 0.000485, epsilon: 0.010001, episode:  216\n",
      "frames: 424000, reward: -4.500000, loss: 0.001165, epsilon: 0.010001, episode:  216\n",
      "frames: 425000, reward: -3.700000, loss: 0.002131, epsilon: 0.010001, episode:  217\n",
      "frames: 426000, reward: -3.700000, loss: 0.001510, epsilon: 0.010001, episode:  217\n",
      "frames: 427000, reward: -3.700000, loss: 0.000639, epsilon: 0.010001, episode:  217\n",
      "frames: 428000, reward: -3.700000, loss: 0.000392, epsilon: 0.010001, episode:  217\n",
      "frames: 429000, reward: -2.600000, loss: 0.001015, epsilon: 0.010001, episode:  218\n",
      "frames: 430000, reward: -2.600000, loss: 0.000669, epsilon: 0.010001, episode:  218\n",
      "frames: 431000, reward: -2.600000, loss: 0.001875, epsilon: 0.010001, episode:  218\n",
      "frames: 432000, reward: -2.600000, loss: 0.001888, epsilon: 0.010001, episode:  218\n",
      "frames: 433000, reward: -2.600000, loss: 0.000679, epsilon: 0.010001, episode:  218\n",
      "frames: 434000, reward: -1.600000, loss: 0.000863, epsilon: 0.010001, episode:  219\n",
      "frames: 435000, reward: -1.600000, loss: 0.001177, epsilon: 0.010000, episode:  219\n",
      "frames: 436000, reward: -1.600000, loss: 0.001792, epsilon: 0.010000, episode:  219\n",
      "frames: 437000, reward: -1.600000, loss: 0.001108, epsilon: 0.010000, episode:  219\n",
      "frames: 438000, reward: -1.600000, loss: 0.001267, epsilon: 0.010000, episode:  219\n",
      "frames: 439000, reward: 0.300000, loss: 0.003533, epsilon: 0.010000, episode:  220\n",
      "frames: 440000, reward: 0.300000, loss: 0.000724, epsilon: 0.010000, episode:  220\n",
      "frames: 441000, reward: 0.300000, loss: 0.001709, epsilon: 0.010000, episode:  220\n",
      "frames: 442000, reward: 0.300000, loss: 0.000693, epsilon: 0.010000, episode:  220\n",
      "frames: 443000, reward: 0.900000, loss: 0.001005, epsilon: 0.010000, episode:  221\n",
      "frames: 444000, reward: 0.900000, loss: 0.003010, epsilon: 0.010000, episode:  221\n",
      "frames: 445000, reward: 0.900000, loss: 0.000903, epsilon: 0.010000, episode:  221\n",
      "frames: 446000, reward: 0.900000, loss: 0.000654, epsilon: 0.010000, episode:  221\n",
      "frames: 447000, reward: 0.700000, loss: 0.000703, epsilon: 0.010000, episode:  222\n",
      "frames: 448000, reward: 0.700000, loss: 0.002784, epsilon: 0.010000, episode:  222\n",
      "frames: 449000, reward: 0.700000, loss: 0.002829, epsilon: 0.010000, episode:  222\n",
      "frames: 450000, reward: 0.700000, loss: 0.001108, epsilon: 0.010000, episode:  222\n",
      "frames: 451000, reward: 0.700000, loss: 0.001471, epsilon: 0.010000, episode:  222\n",
      "frames: 452000, reward: 0.700000, loss: 0.001005, epsilon: 0.010000, episode:  222\n",
      "frames: 453000, reward: 1.500000, loss: 0.000535, epsilon: 0.010000, episode:  223\n",
      "frames: 454000, reward: 1.500000, loss: 0.003998, epsilon: 0.010000, episode:  223\n",
      "frames: 455000, reward: 1.500000, loss: 0.001031, epsilon: 0.010000, episode:  223\n",
      "frames: 456000, reward: 0.300000, loss: 0.001040, epsilon: 0.010000, episode:  224\n",
      "frames: 457000, reward: 0.300000, loss: 0.000355, epsilon: 0.010000, episode:  224\n",
      "frames: 458000, reward: 0.300000, loss: 0.000698, epsilon: 0.010000, episode:  224\n",
      "frames: 459000, reward: 0.300000, loss: 0.000817, epsilon: 0.010000, episode:  224\n",
      "frames: 460000, reward: 0.700000, loss: 0.001974, epsilon: 0.010000, episode:  225\n",
      "frames: 461000, reward: 0.700000, loss: 0.001426, epsilon: 0.010000, episode:  225\n",
      "frames: 462000, reward: 0.700000, loss: 0.001082, epsilon: 0.010000, episode:  225\n",
      "frames: 463000, reward: 0.700000, loss: 0.001463, epsilon: 0.010000, episode:  225\n",
      "frames: 464000, reward: 0.700000, loss: 0.000547, epsilon: 0.010000, episode:  225\n",
      "frames: 465000, reward: 0.800000, loss: 0.000346, epsilon: 0.010000, episode:  226\n",
      "frames: 466000, reward: 0.800000, loss: 0.000400, epsilon: 0.010000, episode:  226\n",
      "frames: 467000, reward: 0.800000, loss: 0.000652, epsilon: 0.010000, episode:  226\n",
      "frames: 468000, reward: 0.800000, loss: 0.000717, epsilon: 0.010000, episode:  226\n",
      "frames: 469000, reward: 0.900000, loss: 0.000794, epsilon: 0.010000, episode:  227\n",
      "frames: 470000, reward: 0.900000, loss: 0.000370, epsilon: 0.010000, episode:  227\n",
      "frames: 471000, reward: 2.000000, loss: 0.000456, epsilon: 0.010000, episode:  228\n",
      "frames: 472000, reward: 2.000000, loss: 0.001083, epsilon: 0.010000, episode:  228\n",
      "frames: 473000, reward: 2.000000, loss: 0.000798, epsilon: 0.010000, episode:  228\n",
      "frames: 474000, reward: 2.900000, loss: 0.001453, epsilon: 0.010000, episode:  229\n",
      "frames: 475000, reward: 2.900000, loss: 0.001387, epsilon: 0.010000, episode:  229\n",
      "frames: 476000, reward: 2.900000, loss: 0.000894, epsilon: 0.010000, episode:  229\n",
      "frames: 477000, reward: 3.900000, loss: 0.003110, epsilon: 0.010000, episode:  230\n",
      "frames: 478000, reward: 3.900000, loss: 0.000908, epsilon: 0.010000, episode:  230\n",
      "frames: 479000, reward: 5.600000, loss: 0.000313, epsilon: 0.010000, episode:  231\n",
      "frames: 480000, reward: 5.600000, loss: 0.001203, epsilon: 0.010000, episode:  231\n",
      "frames: 481000, reward: 5.600000, loss: 0.000519, epsilon: 0.010000, episode:  231\n",
      "frames: 482000, reward: 5.600000, loss: 0.000742, epsilon: 0.010000, episode:  231\n",
      "frames: 483000, reward: 5.600000, loss: 0.000683, epsilon: 0.010000, episode:  231\n",
      "frames: 484000, reward: 5.600000, loss: 0.000944, epsilon: 0.010000, episode:  232\n",
      "frames: 485000, reward: 5.600000, loss: 0.000505, epsilon: 0.010000, episode:  232\n",
      "frames: 486000, reward: 5.600000, loss: 0.000349, epsilon: 0.010000, episode:  232\n",
      "frames: 487000, reward: 6.000000, loss: 0.000526, epsilon: 0.010000, episode:  233\n",
      "frames: 488000, reward: 6.000000, loss: 0.000635, epsilon: 0.010000, episode:  233\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 489000, reward: 8.500000, loss: 0.002598, epsilon: 0.010000, episode:  234\n",
      "frames: 490000, reward: 8.500000, loss: 0.000536, epsilon: 0.010000, episode:  234\n",
      "frames: 491000, reward: 8.500000, loss: 0.000470, epsilon: 0.010000, episode:  234\n",
      "frames: 492000, reward: 9.500000, loss: 0.000641, epsilon: 0.010000, episode:  235\n",
      "frames: 493000, reward: 9.500000, loss: 0.000906, epsilon: 0.010000, episode:  235\n",
      "frames: 494000, reward: 9.500000, loss: 0.000328, epsilon: 0.010000, episode:  235\n",
      "frames: 495000, reward: 11.200000, loss: 0.001238, epsilon: 0.010000, episode:  236\n",
      "frames: 496000, reward: 11.200000, loss: 0.000802, epsilon: 0.010000, episode:  236\n",
      "frames: 497000, reward: 11.200000, loss: 0.000493, epsilon: 0.010000, episode:  236\n",
      "frames: 498000, reward: 12.400000, loss: 0.001505, epsilon: 0.010000, episode:  237\n",
      "frames: 499000, reward: 12.400000, loss: 0.000735, epsilon: 0.010000, episode:  237\n",
      "frames: 500000, reward: 12.700000, loss: 0.000334, epsilon: 0.010000, episode:  238\n",
      "frames: 501000, reward: 12.700000, loss: 0.000628, epsilon: 0.010000, episode:  238\n",
      "frames: 502000, reward: 13.100000, loss: 0.000445, epsilon: 0.010000, episode:  239\n",
      "frames: 503000, reward: 13.800000, loss: 0.000890, epsilon: 0.010000, episode:  240\n",
      "frames: 504000, reward: 13.800000, loss: 0.000806, epsilon: 0.010000, episode:  240\n",
      "frames: 505000, reward: 14.100000, loss: 0.000517, epsilon: 0.010000, episode:  241\n",
      "frames: 506000, reward: 14.100000, loss: 0.000748, epsilon: 0.010000, episode:  241\n",
      "frames: 507000, reward: 14.100000, loss: 0.000458, epsilon: 0.010000, episode:  241\n",
      "frames: 508000, reward: 16.300000, loss: 0.000351, epsilon: 0.010000, episode:  242\n",
      "frames: 509000, reward: 16.300000, loss: 0.000507, epsilon: 0.010000, episode:  242\n",
      "frames: 510000, reward: 17.200000, loss: 0.000555, epsilon: 0.010000, episode:  243\n",
      "frames: 511000, reward: 17.200000, loss: 0.000574, epsilon: 0.010000, episode:  243\n",
      "frames: 512000, reward: 17.200000, loss: 0.000546, epsilon: 0.010000, episode:  243\n",
      "frames: 513000, reward: 16.700000, loss: 0.000310, epsilon: 0.010000, episode:  244\n",
      "frames: 514000, reward: 16.700000, loss: 0.000940, epsilon: 0.010000, episode:  244\n",
      "frames: 515000, reward: 17.100000, loss: 0.000592, epsilon: 0.010000, episode:  245\n",
      "frames: 516000, reward: 17.100000, loss: 0.000321, epsilon: 0.010000, episode:  245\n",
      "frames: 517000, reward: 17.800000, loss: 0.000381, epsilon: 0.010000, episode:  246\n",
      "frames: 518000, reward: 17.800000, loss: 0.000507, epsilon: 0.010000, episode:  246\n",
      "frames: 519000, reward: 17.800000, loss: 0.000545, epsilon: 0.010000, episode:  246\n",
      "frames: 520000, reward: 17.800000, loss: 0.000356, epsilon: 0.010000, episode:  246\n",
      "frames: 521000, reward: 17.300000, loss: 0.000314, epsilon: 0.010000, episode:  247\n",
      "frames: 522000, reward: 17.300000, loss: 0.001265, epsilon: 0.010000, episode:  247\n",
      "frames: 523000, reward: 16.900000, loss: 0.003715, epsilon: 0.010000, episode:  248\n",
      "frames: 524000, reward: 16.900000, loss: 0.000442, epsilon: 0.010000, episode:  248\n",
      "frames: 525000, reward: 17.100000, loss: 0.000262, epsilon: 0.010000, episode:  249\n",
      "frames: 526000, reward: 16.900000, loss: 0.000526, epsilon: 0.010000, episode:  250\n",
      "frames: 527000, reward: 16.900000, loss: 0.000430, epsilon: 0.010000, episode:  250\n",
      "frames: 528000, reward: 16.600000, loss: 0.000538, epsilon: 0.010000, episode:  251\n",
      "frames: 529000, reward: 16.600000, loss: 0.000535, epsilon: 0.010000, episode:  251\n",
      "frames: 530000, reward: 16.700000, loss: 0.000193, epsilon: 0.010000, episode:  252\n",
      "frames: 531000, reward: 16.700000, loss: 0.000269, epsilon: 0.010000, episode:  252\n",
      "frames: 532000, reward: 16.700000, loss: 0.001407, epsilon: 0.010000, episode:  253\n",
      "frames: 533000, reward: 16.700000, loss: 0.001290, epsilon: 0.010000, episode:  253\n",
      "frames: 534000, reward: 17.200000, loss: 0.000255, epsilon: 0.010000, episode:  254\n",
      "frames: 535000, reward: 17.200000, loss: 0.000308, epsilon: 0.010000, episode:  254\n",
      "frames: 536000, reward: 17.400000, loss: 0.000496, epsilon: 0.010000, episode:  255\n",
      "frames: 537000, reward: 17.400000, loss: 0.000444, epsilon: 0.010000, episode:  255\n",
      "frames: 538000, reward: 17.700000, loss: 0.000625, epsilon: 0.010000, episode:  256\n",
      "frames: 539000, reward: 17.700000, loss: 0.000145, epsilon: 0.010000, episode:  256\n",
      "frames: 540000, reward: 17.900000, loss: 0.001142, epsilon: 0.010000, episode:  257\n",
      "frames: 541000, reward: 17.900000, loss: 0.000166, epsilon: 0.010000, episode:  257\n",
      "frames: 542000, reward: 18.300000, loss: 0.000172, epsilon: 0.010000, episode:  258\n",
      "frames: 543000, reward: 18.300000, loss: 0.000175, epsilon: 0.010000, episode:  258\n",
      "frames: 544000, reward: 18.100000, loss: 0.001049, epsilon: 0.010000, episode:  259\n",
      "frames: 545000, reward: 18.100000, loss: 0.001005, epsilon: 0.010000, episode:  259\n",
      "frames: 546000, reward: 18.100000, loss: 0.001421, epsilon: 0.010000, episode:  259\n",
      "frames: 547000, reward: 18.100000, loss: 0.001003, epsilon: 0.010000, episode:  259\n",
      "frames: 548000, reward: 17.300000, loss: 0.000903, epsilon: 0.010000, episode:  260\n",
      "frames: 549000, reward: 17.800000, loss: 0.000423, epsilon: 0.010000, episode:  261\n",
      "frames: 550000, reward: 17.800000, loss: 0.000463, epsilon: 0.010000, episode:  261\n",
      "frames: 551000, reward: 17.700000, loss: 0.000299, epsilon: 0.010000, episode:  262\n",
      "frames: 552000, reward: 17.700000, loss: 0.000461, epsilon: 0.010000, episode:  262\n",
      "frames: 553000, reward: 17.900000, loss: 0.000622, epsilon: 0.010000, episode:  263\n",
      "frames: 554000, reward: 17.900000, loss: 0.001318, epsilon: 0.010000, episode:  263\n",
      "frames: 555000, reward: 18.000000, loss: 0.000687, epsilon: 0.010000, episode:  264\n",
      "frames: 556000, reward: 18.000000, loss: 0.000547, epsilon: 0.010000, episode:  264\n",
      "frames: 557000, reward: 17.300000, loss: 0.004323, epsilon: 0.010000, episode:  265\n",
      "frames: 558000, reward: 17.300000, loss: 0.002107, epsilon: 0.010000, episode:  265\n",
      "frames: 559000, reward: 17.300000, loss: 0.000378, epsilon: 0.010000, episode:  265\n",
      "frames: 560000, reward: 16.900000, loss: 0.000312, epsilon: 0.010000, episode:  266\n",
      "frames: 561000, reward: 16.900000, loss: 0.000680, epsilon: 0.010000, episode:  266\n",
      "frames: 562000, reward: 16.300000, loss: 0.000360, epsilon: 0.010000, episode:  267\n",
      "frames: 563000, reward: 16.300000, loss: 0.000493, epsilon: 0.010000, episode:  267\n",
      "frames: 564000, reward: 16.300000, loss: 0.000355, epsilon: 0.010000, episode:  268\n",
      "frames: 565000, reward: 16.300000, loss: 0.000260, epsilon: 0.010000, episode:  268\n",
      "frames: 566000, reward: 16.700000, loss: 0.000486, epsilon: 0.010000, episode:  269\n",
      "frames: 567000, reward: 16.700000, loss: 0.000194, epsilon: 0.010000, episode:  269\n",
      "frames: 568000, reward: 17.500000, loss: 0.000421, epsilon: 0.010000, episode:  270\n",
      "frames: 569000, reward: 17.500000, loss: 0.000576, epsilon: 0.010000, episode:  270\n",
      "frames: 570000, reward: 17.500000, loss: 0.001007, epsilon: 0.010000, episode:  270\n",
      "frames: 571000, reward: 16.700000, loss: 0.000555, epsilon: 0.010000, episode:  271\n",
      "frames: 572000, reward: 16.700000, loss: 0.000721, epsilon: 0.010000, episode:  271\n",
      "frames: 573000, reward: 16.500000, loss: 0.000155, epsilon: 0.010000, episode:  272\n",
      "frames: 574000, reward: 16.500000, loss: 0.000265, epsilon: 0.010000, episode:  272\n",
      "frames: 575000, reward: 16.300000, loss: 0.000395, epsilon: 0.010000, episode:  273\n",
      "frames: 576000, reward: 16.300000, loss: 0.000233, epsilon: 0.010000, episode:  273\n",
      "frames: 577000, reward: 16.100000, loss: 0.000250, epsilon: 0.010000, episode:  274\n",
      "frames: 578000, reward: 16.100000, loss: 0.001398, epsilon: 0.010000, episode:  274\n",
      "frames: 579000, reward: 16.200000, loss: 0.000834, epsilon: 0.010000, episode:  275\n",
      "frames: 580000, reward: 16.200000, loss: 0.000624, epsilon: 0.010000, episode:  275\n",
      "frames: 581000, reward: 16.100000, loss: 0.000242, epsilon: 0.010000, episode:  276\n",
      "frames: 582000, reward: 16.100000, loss: 0.000203, epsilon: 0.010000, episode:  276\n",
      "frames: 583000, reward: 16.800000, loss: 0.000229, epsilon: 0.010000, episode:  277\n",
      "frames: 584000, reward: 16.800000, loss: 0.000327, epsilon: 0.010000, episode:  277\n",
      "frames: 585000, reward: 16.500000, loss: 0.000328, epsilon: 0.010000, episode:  278\n",
      "frames: 586000, reward: 16.500000, loss: 0.000935, epsilon: 0.010000, episode:  278\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 587000, reward: 16.500000, loss: 0.001485, epsilon: 0.010000, episode:  278\n",
      "frames: 588000, reward: 15.500000, loss: 0.000290, epsilon: 0.010000, episode:  279\n",
      "frames: 589000, reward: 15.500000, loss: 0.000884, epsilon: 0.010000, episode:  279\n",
      "frames: 590000, reward: 15.500000, loss: 0.001651, epsilon: 0.010000, episode:  279\n",
      "frames: 591000, reward: 15.400000, loss: 0.000676, epsilon: 0.010000, episode:  280\n",
      "frames: 592000, reward: 15.400000, loss: 0.000467, epsilon: 0.010000, episode:  280\n",
      "frames: 593000, reward: 15.600000, loss: 0.000473, epsilon: 0.010000, episode:  281\n",
      "frames: 594000, reward: 15.600000, loss: 0.000706, epsilon: 0.010000, episode:  281\n",
      "frames: 595000, reward: 15.600000, loss: 0.000821, epsilon: 0.010000, episode:  281\n",
      "frames: 596000, reward: 15.300000, loss: 0.000919, epsilon: 0.010000, episode:  282\n",
      "frames: 597000, reward: 15.300000, loss: 0.001000, epsilon: 0.010000, episode:  282\n",
      "frames: 598000, reward: 15.300000, loss: 0.001086, epsilon: 0.010000, episode:  282\n",
      "frames: 599000, reward: 15.300000, loss: 0.000721, epsilon: 0.010000, episode:  282\n",
      "frames: 600000, reward: 13.000000, loss: 0.000705, epsilon: 0.010000, episode:  283\n",
      "frames: 601000, reward: 13.000000, loss: 0.001388, epsilon: 0.010000, episode:  283\n",
      "frames: 602000, reward: 13.000000, loss: 0.002449, epsilon: 0.010000, episode:  283\n",
      "frames: 603000, reward: 12.600000, loss: 0.000597, epsilon: 0.010000, episode:  284\n",
      "frames: 604000, reward: 12.600000, loss: 0.001753, epsilon: 0.010000, episode:  284\n",
      "frames: 605000, reward: 12.800000, loss: 0.001538, epsilon: 0.010000, episode:  285\n",
      "frames: 606000, reward: 12.800000, loss: 0.000850, epsilon: 0.010000, episode:  285\n",
      "frames: 607000, reward: 12.900000, loss: 0.001138, epsilon: 0.010000, episode:  286\n",
      "frames: 608000, reward: 12.900000, loss: 0.001024, epsilon: 0.010000, episode:  286\n",
      "frames: 609000, reward: 12.900000, loss: 0.000578, epsilon: 0.010000, episode:  286\n",
      "frames: 610000, reward: 13.200000, loss: 0.000617, epsilon: 0.010000, episode:  287\n",
      "frames: 611000, reward: 13.200000, loss: 0.001382, epsilon: 0.010000, episode:  287\n",
      "frames: 612000, reward: 13.200000, loss: 0.002500, epsilon: 0.010000, episode:  288\n",
      "frames: 613000, reward: 13.200000, loss: 0.000654, epsilon: 0.010000, episode:  288\n",
      "frames: 614000, reward: 13.200000, loss: 0.000592, epsilon: 0.010000, episode:  288\n",
      "frames: 615000, reward: 13.200000, loss: 0.000565, epsilon: 0.010000, episode:  288\n",
      "frames: 616000, reward: 13.600000, loss: 0.000631, epsilon: 0.010000, episode:  289\n",
      "frames: 617000, reward: 13.600000, loss: 0.000786, epsilon: 0.010000, episode:  289\n",
      "frames: 618000, reward: 13.600000, loss: 0.000381, epsilon: 0.010000, episode:  290\n",
      "frames: 619000, reward: 13.600000, loss: 0.000370, epsilon: 0.010000, episode:  290\n",
      "frames: 620000, reward: 13.900000, loss: 0.000542, epsilon: 0.010000, episode:  291\n",
      "frames: 621000, reward: 13.900000, loss: 0.000777, epsilon: 0.010000, episode:  291\n",
      "frames: 622000, reward: 13.900000, loss: 0.001648, epsilon: 0.010000, episode:  291\n",
      "frames: 623000, reward: 13.500000, loss: 0.000276, epsilon: 0.010000, episode:  292\n",
      "frames: 624000, reward: 13.500000, loss: 0.000299, epsilon: 0.010000, episode:  292\n",
      "frames: 625000, reward: 15.800000, loss: 0.000346, epsilon: 0.010000, episode:  293\n",
      "frames: 626000, reward: 15.800000, loss: 0.000824, epsilon: 0.010000, episode:  293\n",
      "frames: 627000, reward: 16.200000, loss: 0.000511, epsilon: 0.010000, episode:  294\n",
      "frames: 628000, reward: 16.200000, loss: 0.000514, epsilon: 0.010000, episode:  294\n",
      "frames: 629000, reward: 16.200000, loss: 0.000829, epsilon: 0.010000, episode:  295\n",
      "frames: 630000, reward: 16.200000, loss: 0.000265, epsilon: 0.010000, episode:  295\n",
      "frames: 631000, reward: 16.300000, loss: 0.000768, epsilon: 0.010000, episode:  296\n",
      "frames: 632000, reward: 16.300000, loss: 0.000441, epsilon: 0.010000, episode:  296\n",
      "frames: 633000, reward: 16.300000, loss: 0.000606, epsilon: 0.010000, episode:  296\n",
      "frames: 634000, reward: 16.000000, loss: 0.000265, epsilon: 0.010000, episode:  297\n",
      "frames: 635000, reward: 16.400000, loss: 0.000758, epsilon: 0.010000, episode:  298\n",
      "frames: 636000, reward: 16.400000, loss: 0.000881, epsilon: 0.010000, episode:  298\n",
      "frames: 637000, reward: 16.400000, loss: 0.000638, epsilon: 0.010000, episode:  298\n",
      "frames: 638000, reward: 15.900000, loss: 0.000733, epsilon: 0.010000, episode:  299\n",
      "frames: 639000, reward: 15.900000, loss: 0.000354, epsilon: 0.010000, episode:  299\n",
      "frames: 640000, reward: 15.900000, loss: 0.000598, epsilon: 0.010000, episode:  300\n",
      "frames: 641000, reward: 15.900000, loss: 0.001517, epsilon: 0.010000, episode:  300\n",
      "frames: 642000, reward: 15.900000, loss: 0.000332, epsilon: 0.010000, episode:  301\n",
      "frames: 643000, reward: 15.900000, loss: 0.000228, epsilon: 0.010000, episode:  301\n",
      "frames: 644000, reward: 15.900000, loss: 0.000388, epsilon: 0.010000, episode:  301\n",
      "frames: 645000, reward: 17.000000, loss: 0.000337, epsilon: 0.010000, episode:  302\n",
      "frames: 646000, reward: 17.000000, loss: 0.000348, epsilon: 0.010000, episode:  302\n",
      "frames: 647000, reward: 17.200000, loss: 0.000257, epsilon: 0.010000, episode:  303\n",
      "frames: 648000, reward: 17.600000, loss: 0.000213, epsilon: 0.010000, episode:  304\n",
      "frames: 649000, reward: 17.600000, loss: 0.000401, epsilon: 0.010000, episode:  304\n",
      "frames: 650000, reward: 17.700000, loss: 0.001307, epsilon: 0.010000, episode:  305\n",
      "frames: 651000, reward: 17.700000, loss: 0.000762, epsilon: 0.010000, episode:  305\n",
      "frames: 652000, reward: 17.900000, loss: 0.000162, epsilon: 0.010000, episode:  306\n",
      "frames: 653000, reward: 17.900000, loss: 0.000302, epsilon: 0.010000, episode:  306\n",
      "frames: 654000, reward: 17.900000, loss: 0.000309, epsilon: 0.010000, episode:  306\n",
      "frames: 655000, reward: 17.900000, loss: 0.000680, epsilon: 0.010000, episode:  306\n",
      "frames: 656000, reward: 17.900000, loss: 0.000442, epsilon: 0.010000, episode:  307\n",
      "frames: 657000, reward: 17.900000, loss: 0.000582, epsilon: 0.010000, episode:  307\n",
      "frames: 658000, reward: 17.900000, loss: 0.000619, epsilon: 0.010000, episode:  307\n",
      "frames: 659000, reward: 16.500000, loss: 0.000630, epsilon: 0.010000, episode:  308\n",
      "frames: 660000, reward: 16.500000, loss: 0.000705, epsilon: 0.010000, episode:  308\n",
      "frames: 661000, reward: 17.400000, loss: 0.000475, epsilon: 0.010000, episode:  309\n",
      "frames: 662000, reward: 17.400000, loss: 0.000406, epsilon: 0.010000, episode:  309\n",
      "frames: 663000, reward: 17.100000, loss: 0.000694, epsilon: 0.010000, episode:  310\n",
      "frames: 664000, reward: 17.100000, loss: 0.000374, epsilon: 0.010000, episode:  310\n",
      "frames: 665000, reward: 17.200000, loss: 0.000333, epsilon: 0.010000, episode:  311\n",
      "frames: 666000, reward: 17.200000, loss: 0.000440, epsilon: 0.010000, episode:  311\n",
      "frames: 667000, reward: 17.000000, loss: 0.000478, epsilon: 0.010000, episode:  312\n",
      "frames: 668000, reward: 17.000000, loss: 0.000613, epsilon: 0.010000, episode:  312\n",
      "frames: 669000, reward: 16.900000, loss: 0.000445, epsilon: 0.010000, episode:  313\n",
      "frames: 670000, reward: 16.900000, loss: 0.000876, epsilon: 0.010000, episode:  313\n",
      "frames: 671000, reward: 16.700000, loss: 0.000725, epsilon: 0.010000, episode:  314\n",
      "frames: 672000, reward: 16.700000, loss: 0.000478, epsilon: 0.010000, episode:  314\n",
      "frames: 673000, reward: 16.900000, loss: 0.000310, epsilon: 0.010000, episode:  315\n",
      "frames: 674000, reward: 16.900000, loss: 0.000428, epsilon: 0.010000, episode:  315\n",
      "frames: 675000, reward: 16.500000, loss: 0.001291, epsilon: 0.010000, episode:  316\n",
      "frames: 676000, reward: 16.500000, loss: 0.000360, epsilon: 0.010000, episode:  316\n",
      "frames: 677000, reward: 16.800000, loss: 0.000227, epsilon: 0.010000, episode:  317\n",
      "frames: 678000, reward: 16.800000, loss: 0.000358, epsilon: 0.010000, episode:  317\n",
      "frames: 679000, reward: 18.100000, loss: 0.002144, epsilon: 0.010000, episode:  318\n",
      "frames: 680000, reward: 18.100000, loss: 0.000242, epsilon: 0.010000, episode:  318\n",
      "frames: 681000, reward: 18.200000, loss: 0.000231, epsilon: 0.010000, episode:  319\n",
      "frames: 682000, reward: 18.200000, loss: 0.000138, epsilon: 0.010000, episode:  319\n",
      "frames: 683000, reward: 18.400000, loss: 0.003560, epsilon: 0.010000, episode:  320\n",
      "frames: 684000, reward: 18.400000, loss: 0.000358, epsilon: 0.010000, episode:  320\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 685000, reward: 18.400000, loss: 0.000253, epsilon: 0.010000, episode:  321\n",
      "frames: 686000, reward: 18.400000, loss: 0.000538, epsilon: 0.010000, episode:  321\n",
      "frames: 687000, reward: 18.400000, loss: 0.000131, epsilon: 0.010000, episode:  322\n",
      "frames: 688000, reward: 18.400000, loss: 0.001546, epsilon: 0.010000, episode:  322\n",
      "frames: 689000, reward: 18.200000, loss: 0.000154, epsilon: 0.010000, episode:  323\n",
      "frames: 690000, reward: 18.300000, loss: 0.001711, epsilon: 0.010000, episode:  324\n",
      "frames: 691000, reward: 18.300000, loss: 0.000168, epsilon: 0.010000, episode:  324\n",
      "frames: 692000, reward: 18.100000, loss: 0.000249, epsilon: 0.010000, episode:  325\n",
      "frames: 693000, reward: 18.100000, loss: 0.000293, epsilon: 0.010000, episode:  325\n",
      "frames: 694000, reward: 18.100000, loss: 0.000208, epsilon: 0.010000, episode:  325\n",
      "frames: 695000, reward: 18.200000, loss: 0.000662, epsilon: 0.010000, episode:  326\n",
      "frames: 696000, reward: 18.300000, loss: 0.000314, epsilon: 0.010000, episode:  327\n",
      "frames: 697000, reward: 18.300000, loss: 0.001114, epsilon: 0.010000, episode:  327\n",
      "frames: 698000, reward: 18.300000, loss: 0.000729, epsilon: 0.010000, episode:  327\n",
      "frames: 699000, reward: 18.200000, loss: 0.000229, epsilon: 0.010000, episode:  328\n",
      "frames: 700000, reward: 18.300000, loss: 0.000210, epsilon: 0.010000, episode:  329\n",
      "frames: 701000, reward: 18.300000, loss: 0.000151, epsilon: 0.010000, episode:  329\n",
      "frames: 702000, reward: 18.600000, loss: 0.000607, epsilon: 0.010000, episode:  330\n",
      "frames: 703000, reward: 18.600000, loss: 0.000150, epsilon: 0.010000, episode:  330\n",
      "frames: 704000, reward: 18.600000, loss: 0.001171, epsilon: 0.010000, episode:  331\n",
      "frames: 705000, reward: 18.600000, loss: 0.000755, epsilon: 0.010000, episode:  331\n",
      "frames: 706000, reward: 18.400000, loss: 0.000213, epsilon: 0.010000, episode:  332\n",
      "frames: 707000, reward: 18.400000, loss: 0.000431, epsilon: 0.010000, episode:  332\n",
      "frames: 708000, reward: 18.600000, loss: 0.000305, epsilon: 0.010000, episode:  333\n",
      "frames: 709000, reward: 18.600000, loss: 0.000307, epsilon: 0.010000, episode:  333\n",
      "frames: 710000, reward: 18.600000, loss: 0.000485, epsilon: 0.010000, episode:  333\n",
      "frames: 711000, reward: 18.500000, loss: 0.000237, epsilon: 0.010000, episode:  334\n",
      "frames: 712000, reward: 18.500000, loss: 0.000248, epsilon: 0.010000, episode:  334\n",
      "frames: 713000, reward: 18.500000, loss: 0.000855, epsilon: 0.010000, episode:  335\n",
      "frames: 714000, reward: 18.500000, loss: 0.000346, epsilon: 0.010000, episode:  335\n",
      "frames: 715000, reward: 18.400000, loss: 0.000269, epsilon: 0.010000, episode:  336\n",
      "frames: 716000, reward: 18.400000, loss: 0.000505, epsilon: 0.010000, episode:  336\n",
      "frames: 717000, reward: 18.500000, loss: 0.000368, epsilon: 0.010000, episode:  337\n",
      "frames: 718000, reward: 18.500000, loss: 0.000294, epsilon: 0.010000, episode:  337\n",
      "frames: 719000, reward: 18.300000, loss: 0.000388, epsilon: 0.010000, episode:  338\n",
      "frames: 720000, reward: 18.300000, loss: 0.000281, epsilon: 0.010000, episode:  338\n",
      "frames: 721000, reward: 17.500000, loss: 0.000397, epsilon: 0.010000, episode:  339\n",
      "frames: 722000, reward: 17.500000, loss: 0.000217, epsilon: 0.010000, episode:  339\n",
      "frames: 723000, reward: 17.500000, loss: 0.000241, epsilon: 0.010000, episode:  340\n",
      "frames: 724000, reward: 17.500000, loss: 0.000251, epsilon: 0.010000, episode:  340\n",
      "frames: 725000, reward: 17.500000, loss: 0.000419, epsilon: 0.010000, episode:  341\n",
      "frames: 726000, reward: 17.500000, loss: 0.000773, epsilon: 0.010000, episode:  341\n",
      "frames: 727000, reward: 17.600000, loss: 0.000160, epsilon: 0.010000, episode:  342\n",
      "frames: 728000, reward: 17.600000, loss: 0.000476, epsilon: 0.010000, episode:  342\n",
      "frames: 729000, reward: 17.600000, loss: 0.000430, epsilon: 0.010000, episode:  343\n",
      "frames: 730000, reward: 17.600000, loss: 0.000274, epsilon: 0.010000, episode:  343\n",
      "frames: 731000, reward: 17.800000, loss: 0.000126, epsilon: 0.010000, episode:  344\n",
      "frames: 732000, reward: 18.000000, loss: 0.000806, epsilon: 0.010000, episode:  345\n",
      "frames: 733000, reward: 18.000000, loss: 0.000189, epsilon: 0.010000, episode:  345\n",
      "frames: 734000, reward: 18.000000, loss: 0.000207, epsilon: 0.010000, episode:  345\n",
      "frames: 735000, reward: 18.000000, loss: 0.000160, epsilon: 0.010000, episode:  346\n",
      "frames: 736000, reward: 18.000000, loss: 0.000275, epsilon: 0.010000, episode:  346\n",
      "frames: 737000, reward: 17.800000, loss: 0.000231, epsilon: 0.010000, episode:  347\n",
      "frames: 738000, reward: 17.800000, loss: 0.000126, epsilon: 0.010000, episode:  347\n",
      "frames: 739000, reward: 18.000000, loss: 0.000135, epsilon: 0.010000, episode:  348\n",
      "frames: 740000, reward: 18.000000, loss: 0.000576, epsilon: 0.010000, episode:  348\n",
      "frames: 741000, reward: 18.700000, loss: 0.000120, epsilon: 0.010000, episode:  349\n",
      "frames: 742000, reward: 18.700000, loss: 0.000760, epsilon: 0.010000, episode:  349\n",
      "frames: 743000, reward: 18.200000, loss: 0.000587, epsilon: 0.010000, episode:  350\n",
      "frames: 744000, reward: 18.200000, loss: 0.000274, epsilon: 0.010000, episode:  350\n",
      "frames: 745000, reward: 18.200000, loss: 0.000166, epsilon: 0.010000, episode:  350\n",
      "frames: 746000, reward: 17.800000, loss: 0.000797, epsilon: 0.010000, episode:  351\n",
      "frames: 747000, reward: 17.800000, loss: 0.000224, epsilon: 0.010000, episode:  351\n",
      "frames: 748000, reward: 17.400000, loss: 0.000240, epsilon: 0.010000, episode:  352\n",
      "frames: 749000, reward: 17.400000, loss: 0.000306, epsilon: 0.010000, episode:  352\n",
      "frames: 750000, reward: 17.700000, loss: 0.000389, epsilon: 0.010000, episode:  353\n",
      "frames: 751000, reward: 17.700000, loss: 0.000304, epsilon: 0.010000, episode:  353\n",
      "frames: 752000, reward: 17.300000, loss: 0.000294, epsilon: 0.010000, episode:  354\n",
      "frames: 753000, reward: 17.300000, loss: 0.000310, epsilon: 0.010000, episode:  354\n",
      "frames: 754000, reward: 17.200000, loss: 0.000355, epsilon: 0.010000, episode:  355\n",
      "frames: 755000, reward: 17.200000, loss: 0.000329, epsilon: 0.010000, episode:  355\n",
      "frames: 756000, reward: 17.700000, loss: 0.000567, epsilon: 0.010000, episode:  356\n",
      "frames: 757000, reward: 17.700000, loss: 0.000092, epsilon: 0.010000, episode:  356\n",
      "frames: 758000, reward: 17.700000, loss: 0.000253, epsilon: 0.010000, episode:  357\n",
      "frames: 759000, reward: 17.700000, loss: 0.000418, epsilon: 0.010000, episode:  357\n",
      "frames: 760000, reward: 17.800000, loss: 0.000171, epsilon: 0.010000, episode:  358\n",
      "frames: 761000, reward: 17.800000, loss: 0.000184, epsilon: 0.010000, episode:  358\n",
      "frames: 762000, reward: 17.900000, loss: 0.000300, epsilon: 0.010000, episode:  359\n",
      "frames: 763000, reward: 17.900000, loss: 0.000212, epsilon: 0.010000, episode:  359\n",
      "frames: 764000, reward: 18.400000, loss: 0.000255, epsilon: 0.010000, episode:  360\n",
      "frames: 765000, reward: 18.400000, loss: 0.000301, epsilon: 0.010000, episode:  360\n",
      "frames: 766000, reward: 18.500000, loss: 0.000315, epsilon: 0.010000, episode:  361\n",
      "frames: 767000, reward: 19.100000, loss: 0.000266, epsilon: 0.010000, episode:  362\n",
      "frames: 768000, reward: 19.100000, loss: 0.000149, epsilon: 0.010000, episode:  362\n",
      "frames: 769000, reward: 19.100000, loss: 0.000891, epsilon: 0.010000, episode:  363\n",
      "frames: 770000, reward: 19.100000, loss: 0.000499, epsilon: 0.010000, episode:  363\n",
      "frames: 771000, reward: 19.500000, loss: 0.000388, epsilon: 0.010000, episode:  364\n",
      "frames: 772000, reward: 19.500000, loss: 0.000180, epsilon: 0.010000, episode:  364\n",
      "frames: 773000, reward: 19.600000, loss: 0.000119, epsilon: 0.010000, episode:  365\n",
      "frames: 774000, reward: 19.600000, loss: 0.000205, epsilon: 0.010000, episode:  365\n",
      "frames: 775000, reward: 19.200000, loss: 0.000307, epsilon: 0.010000, episode:  366\n",
      "frames: 776000, reward: 19.200000, loss: 0.000605, epsilon: 0.010000, episode:  366\n",
      "frames: 777000, reward: 19.000000, loss: 0.000189, epsilon: 0.010000, episode:  367\n",
      "frames: 778000, reward: 19.000000, loss: 0.000426, epsilon: 0.010000, episode:  367\n",
      "frames: 779000, reward: 19.000000, loss: 0.000357, epsilon: 0.010000, episode:  368\n",
      "frames: 780000, reward: 19.000000, loss: 0.000168, epsilon: 0.010000, episode:  368\n",
      "frames: 781000, reward: 18.500000, loss: 0.000341, epsilon: 0.010000, episode:  369\n",
      "frames: 782000, reward: 18.500000, loss: 0.000163, epsilon: 0.010000, episode:  369\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 783000, reward: 18.600000, loss: 0.000165, epsilon: 0.010000, episode:  370\n",
      "frames: 784000, reward: 18.600000, loss: 0.000514, epsilon: 0.010000, episode:  370\n",
      "frames: 785000, reward: 18.600000, loss: 0.000527, epsilon: 0.010000, episode:  371\n",
      "frames: 786000, reward: 18.600000, loss: 0.000108, epsilon: 0.010000, episode:  371\n",
      "frames: 787000, reward: 18.700000, loss: 0.000745, epsilon: 0.010000, episode:  372\n",
      "frames: 788000, reward: 18.700000, loss: 0.000194, epsilon: 0.010000, episode:  372\n",
      "frames: 789000, reward: 18.600000, loss: 0.000321, epsilon: 0.010000, episode:  373\n",
      "frames: 790000, reward: 18.600000, loss: 0.000712, epsilon: 0.010000, episode:  373\n",
      "frames: 791000, reward: 18.300000, loss: 0.000375, epsilon: 0.010000, episode:  374\n",
      "frames: 792000, reward: 18.300000, loss: 0.000348, epsilon: 0.010000, episode:  374\n",
      "frames: 793000, reward: 18.400000, loss: 0.000216, epsilon: 0.010000, episode:  375\n",
      "frames: 794000, reward: 18.400000, loss: 0.000233, epsilon: 0.010000, episode:  375\n",
      "frames: 795000, reward: 18.400000, loss: 0.000427, epsilon: 0.010000, episode:  375\n",
      "frames: 796000, reward: 17.300000, loss: 0.000230, epsilon: 0.010000, episode:  376\n",
      "frames: 797000, reward: 17.300000, loss: 0.000451, epsilon: 0.010000, episode:  376\n",
      "frames: 798000, reward: 17.300000, loss: 0.000447, epsilon: 0.010000, episode:  377\n",
      "frames: 799000, reward: 17.300000, loss: 0.000228, epsilon: 0.010000, episode:  377\n",
      "frames: 800000, reward: 17.300000, loss: 0.000682, epsilon: 0.010000, episode:  378\n",
      "frames: 801000, reward: 17.300000, loss: 0.000258, epsilon: 0.010000, episode:  378\n",
      "frames: 802000, reward: 17.600000, loss: 0.000185, epsilon: 0.010000, episode:  379\n",
      "frames: 803000, reward: 17.600000, loss: 0.001090, epsilon: 0.010000, episode:  379\n",
      "frames: 804000, reward: 17.200000, loss: 0.000330, epsilon: 0.010000, episode:  380\n",
      "frames: 805000, reward: 17.200000, loss: 0.000457, epsilon: 0.010000, episode:  380\n",
      "frames: 806000, reward: 17.600000, loss: 0.000742, epsilon: 0.010000, episode:  381\n",
      "frames: 807000, reward: 17.600000, loss: 0.000438, epsilon: 0.010000, episode:  381\n",
      "frames: 808000, reward: 17.100000, loss: 0.000173, epsilon: 0.010000, episode:  382\n",
      "frames: 809000, reward: 17.100000, loss: 0.000160, epsilon: 0.010000, episode:  382\n",
      "frames: 810000, reward: 17.000000, loss: 0.000259, epsilon: 0.010000, episode:  383\n",
      "frames: 811000, reward: 17.000000, loss: 0.000302, epsilon: 0.010000, episode:  383\n",
      "frames: 812000, reward: 17.100000, loss: 0.000293, epsilon: 0.010000, episode:  384\n",
      "frames: 813000, reward: 17.100000, loss: 0.000127, epsilon: 0.010000, episode:  384\n",
      "frames: 814000, reward: 16.600000, loss: 0.000417, epsilon: 0.010000, episode:  385\n",
      "frames: 815000, reward: 16.600000, loss: 0.000262, epsilon: 0.010000, episode:  385\n",
      "frames: 816000, reward: 17.500000, loss: 0.000373, epsilon: 0.010000, episode:  386\n",
      "frames: 817000, reward: 17.500000, loss: 0.000314, epsilon: 0.010000, episode:  386\n",
      "frames: 818000, reward: 17.800000, loss: 0.000545, epsilon: 0.010000, episode:  387\n",
      "frames: 819000, reward: 17.800000, loss: 0.000293, epsilon: 0.010000, episode:  387\n",
      "frames: 820000, reward: 17.800000, loss: 0.000496, epsilon: 0.010000, episode:  387\n",
      "frames: 821000, reward: 17.200000, loss: 0.000530, epsilon: 0.010000, episode:  388\n",
      "frames: 822000, reward: 17.200000, loss: 0.000467, epsilon: 0.010000, episode:  388\n",
      "frames: 823000, reward: 16.900000, loss: 0.000239, epsilon: 0.010000, episode:  389\n",
      "frames: 824000, reward: 16.900000, loss: 0.000201, epsilon: 0.010000, episode:  389\n",
      "frames: 825000, reward: 17.300000, loss: 0.000273, epsilon: 0.010000, episode:  390\n",
      "frames: 826000, reward: 17.300000, loss: 0.000077, epsilon: 0.010000, episode:  390\n",
      "frames: 827000, reward: 17.200000, loss: 0.000479, epsilon: 0.010000, episode:  391\n",
      "frames: 828000, reward: 17.200000, loss: 0.000806, epsilon: 0.010000, episode:  391\n",
      "frames: 829000, reward: 17.600000, loss: 0.000327, epsilon: 0.010000, episode:  392\n",
      "frames: 830000, reward: 17.600000, loss: 0.000333, epsilon: 0.010000, episode:  392\n",
      "frames: 831000, reward: 17.400000, loss: 0.000248, epsilon: 0.010000, episode:  393\n",
      "frames: 832000, reward: 17.700000, loss: 0.000218, epsilon: 0.010000, episode:  394\n",
      "frames: 833000, reward: 17.700000, loss: 0.000276, epsilon: 0.010000, episode:  394\n",
      "frames: 834000, reward: 17.700000, loss: 0.000242, epsilon: 0.010000, episode:  394\n",
      "frames: 835000, reward: 17.500000, loss: 0.000276, epsilon: 0.010000, episode:  395\n",
      "frames: 836000, reward: 17.500000, loss: 0.000930, epsilon: 0.010000, episode:  395\n",
      "frames: 837000, reward: 17.800000, loss: 0.000295, epsilon: 0.010000, episode:  396\n",
      "frames: 838000, reward: 17.800000, loss: 0.000392, epsilon: 0.010000, episode:  396\n",
      "frames: 839000, reward: 17.900000, loss: 0.000416, epsilon: 0.010000, episode:  397\n",
      "frames: 840000, reward: 17.900000, loss: 0.000290, epsilon: 0.010000, episode:  397\n",
      "frames: 841000, reward: 18.400000, loss: 0.000385, epsilon: 0.010000, episode:  398\n",
      "frames: 842000, reward: 18.900000, loss: 0.000273, epsilon: 0.010000, episode:  399\n",
      "frames: 843000, reward: 18.900000, loss: 0.000289, epsilon: 0.010000, episode:  399\n",
      "frames: 844000, reward: 18.600000, loss: 0.000290, epsilon: 0.010000, episode:  400\n",
      "frames: 845000, reward: 18.600000, loss: 0.000306, epsilon: 0.010000, episode:  400\n",
      "frames: 846000, reward: 18.500000, loss: 0.000123, epsilon: 0.010000, episode:  401\n",
      "frames: 847000, reward: 18.500000, loss: 0.000260, epsilon: 0.010000, episode:  401\n",
      "frames: 848000, reward: 18.400000, loss: 0.000146, epsilon: 0.010000, episode:  402\n",
      "frames: 849000, reward: 18.400000, loss: 0.000104, epsilon: 0.010000, episode:  402\n",
      "frames: 850000, reward: 18.800000, loss: 0.000193, epsilon: 0.010000, episode:  403\n",
      "frames: 851000, reward: 18.800000, loss: 0.000163, epsilon: 0.010000, episode:  404\n",
      "frames: 852000, reward: 18.800000, loss: 0.000196, epsilon: 0.010000, episode:  404\n",
      "frames: 853000, reward: 19.400000, loss: 0.000257, epsilon: 0.010000, episode:  405\n",
      "frames: 854000, reward: 19.400000, loss: 0.000135, epsilon: 0.010000, episode:  405\n",
      "frames: 855000, reward: 19.600000, loss: 0.000177, epsilon: 0.010000, episode:  406\n",
      "frames: 856000, reward: 19.600000, loss: 0.000134, epsilon: 0.010000, episode:  406\n",
      "frames: 857000, reward: 19.400000, loss: 0.000411, epsilon: 0.010000, episode:  407\n",
      "frames: 858000, reward: 19.400000, loss: 0.000163, epsilon: 0.010000, episode:  407\n",
      "frames: 859000, reward: 19.600000, loss: 0.000071, epsilon: 0.010000, episode:  408\n",
      "frames: 860000, reward: 19.500000, loss: 0.000608, epsilon: 0.010000, episode:  409\n",
      "frames: 861000, reward: 19.500000, loss: 0.000239, epsilon: 0.010000, episode:  409\n",
      "frames: 862000, reward: 19.500000, loss: 0.000657, epsilon: 0.010000, episode:  409\n",
      "frames: 863000, reward: 19.200000, loss: 0.002403, epsilon: 0.010000, episode:  410\n",
      "frames: 864000, reward: 19.200000, loss: 0.000290, epsilon: 0.010000, episode:  410\n",
      "frames: 865000, reward: 19.400000, loss: 0.000147, epsilon: 0.010000, episode:  411\n",
      "frames: 866000, reward: 19.600000, loss: 0.000232, epsilon: 0.010000, episode:  412\n",
      "frames: 867000, reward: 19.600000, loss: 0.000302, epsilon: 0.010000, episode:  412\n",
      "frames: 868000, reward: 19.500000, loss: 0.000216, epsilon: 0.010000, episode:  413\n",
      "frames: 869000, reward: 19.500000, loss: 0.000110, epsilon: 0.010000, episode:  413\n",
      "frames: 870000, reward: 19.500000, loss: 0.000218, epsilon: 0.010000, episode:  414\n",
      "frames: 871000, reward: 19.500000, loss: 0.000213, epsilon: 0.010000, episode:  414\n",
      "frames: 872000, reward: 19.600000, loss: 0.000239, epsilon: 0.010000, episode:  415\n",
      "frames: 873000, reward: 19.600000, loss: 0.000262, epsilon: 0.010000, episode:  415\n",
      "frames: 874000, reward: 19.500000, loss: 0.000168, epsilon: 0.010000, episode:  416\n",
      "frames: 875000, reward: 19.500000, loss: 0.000092, epsilon: 0.010000, episode:  416\n",
      "frames: 876000, reward: 19.300000, loss: 0.000167, epsilon: 0.010000, episode:  417\n",
      "frames: 877000, reward: 19.200000, loss: 0.000383, epsilon: 0.010000, episode:  418\n",
      "frames: 878000, reward: 19.200000, loss: 0.000173, epsilon: 0.010000, episode:  418\n",
      "frames: 879000, reward: 19.300000, loss: 0.000551, epsilon: 0.010000, episode:  419\n",
      "frames: 880000, reward: 19.300000, loss: 0.000450, epsilon: 0.010000, episode:  419\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 881000, reward: 19.700000, loss: 0.000234, epsilon: 0.010000, episode:  420\n",
      "frames: 882000, reward: 19.700000, loss: 0.000177, epsilon: 0.010000, episode:  420\n",
      "frames: 883000, reward: 19.700000, loss: 0.000112, epsilon: 0.010000, episode:  421\n",
      "frames: 884000, reward: 19.700000, loss: 0.000110, epsilon: 0.010000, episode:  421\n",
      "frames: 885000, reward: 19.300000, loss: 0.000363, epsilon: 0.010000, episode:  422\n",
      "frames: 886000, reward: 19.300000, loss: 0.000177, epsilon: 0.010000, episode:  422\n",
      "frames: 887000, reward: 18.900000, loss: 0.000208, epsilon: 0.010000, episode:  423\n",
      "frames: 888000, reward: 18.900000, loss: 0.000252, epsilon: 0.010000, episode:  423\n",
      "frames: 889000, reward: 18.700000, loss: 0.000403, epsilon: 0.010000, episode:  424\n",
      "frames: 890000, reward: 18.700000, loss: 0.000202, epsilon: 0.010000, episode:  424\n",
      "frames: 891000, reward: 18.600000, loss: 0.000375, epsilon: 0.010000, episode:  425\n",
      "frames: 892000, reward: 18.600000, loss: 0.000266, epsilon: 0.010000, episode:  425\n",
      "frames: 893000, reward: 18.700000, loss: 0.001311, epsilon: 0.010000, episode:  426\n",
      "frames: 894000, reward: 18.700000, loss: 0.001247, epsilon: 0.010000, episode:  426\n",
      "frames: 895000, reward: 18.500000, loss: 0.000235, epsilon: 0.010000, episode:  427\n",
      "frames: 896000, reward: 18.500000, loss: 0.000312, epsilon: 0.010000, episode:  427\n",
      "frames: 897000, reward: 18.100000, loss: 0.000399, epsilon: 0.010000, episode:  428\n",
      "frames: 898000, reward: 18.100000, loss: 0.000204, epsilon: 0.010000, episode:  428\n",
      "frames: 899000, reward: 18.000000, loss: 0.000434, epsilon: 0.010000, episode:  429\n",
      "frames: 900000, reward: 18.000000, loss: 0.000120, epsilon: 0.010000, episode:  429\n",
      "frames: 901000, reward: 18.100000, loss: 0.000150, epsilon: 0.010000, episode:  430\n",
      "frames: 902000, reward: 18.100000, loss: 0.000248, epsilon: 0.010000, episode:  430\n",
      "frames: 903000, reward: 18.000000, loss: 0.000118, epsilon: 0.010000, episode:  431\n",
      "frames: 904000, reward: 18.000000, loss: 0.000670, epsilon: 0.010000, episode:  431\n",
      "frames: 905000, reward: 18.100000, loss: 0.000389, epsilon: 0.010000, episode:  432\n",
      "frames: 906000, reward: 18.100000, loss: 0.000395, epsilon: 0.010000, episode:  432\n",
      "frames: 907000, reward: 18.000000, loss: 0.000442, epsilon: 0.010000, episode:  433\n",
      "frames: 908000, reward: 18.000000, loss: 0.000288, epsilon: 0.010000, episode:  433\n",
      "frames: 909000, reward: 18.100000, loss: 0.000246, epsilon: 0.010000, episode:  434\n",
      "frames: 910000, reward: 18.100000, loss: 0.000174, epsilon: 0.010000, episode:  434\n",
      "frames: 911000, reward: 17.900000, loss: 0.000608, epsilon: 0.010000, episode:  435\n",
      "frames: 912000, reward: 17.900000, loss: 0.000419, epsilon: 0.010000, episode:  435\n",
      "frames: 913000, reward: 17.900000, loss: 0.000397, epsilon: 0.010000, episode:  435\n",
      "frames: 914000, reward: 17.000000, loss: 0.000509, epsilon: 0.010000, episode:  436\n",
      "frames: 915000, reward: 17.000000, loss: 0.000290, epsilon: 0.010000, episode:  436\n",
      "frames: 916000, reward: 17.300000, loss: 0.000207, epsilon: 0.010000, episode:  437\n",
      "frames: 917000, reward: 17.300000, loss: 0.000523, epsilon: 0.010000, episode:  437\n",
      "frames: 918000, reward: 17.600000, loss: 0.000369, epsilon: 0.010000, episode:  438\n",
      "frames: 919000, reward: 17.600000, loss: 0.000710, epsilon: 0.010000, episode:  438\n",
      "frames: 920000, reward: 17.700000, loss: 0.000610, epsilon: 0.010000, episode:  439\n",
      "frames: 921000, reward: 17.700000, loss: 0.000597, epsilon: 0.010000, episode:  439\n",
      "frames: 922000, reward: 17.100000, loss: 0.000274, epsilon: 0.010000, episode:  440\n",
      "frames: 923000, reward: 17.100000, loss: 0.000550, epsilon: 0.010000, episode:  440\n",
      "frames: 924000, reward: 17.000000, loss: 0.000822, epsilon: 0.010000, episode:  441\n",
      "frames: 925000, reward: 17.000000, loss: 0.000227, epsilon: 0.010000, episode:  441\n",
      "frames: 926000, reward: 17.200000, loss: 0.000737, epsilon: 0.010000, episode:  442\n",
      "frames: 927000, reward: 17.200000, loss: 0.000315, epsilon: 0.010000, episode:  442\n",
      "frames: 928000, reward: 17.600000, loss: 0.000235, epsilon: 0.010000, episode:  443\n",
      "frames: 929000, reward: 17.600000, loss: 0.000430, epsilon: 0.010000, episode:  443\n",
      "frames: 930000, reward: 17.300000, loss: 0.000286, epsilon: 0.010000, episode:  444\n",
      "frames: 931000, reward: 17.300000, loss: 0.000742, epsilon: 0.010000, episode:  444\n",
      "frames: 932000, reward: 17.300000, loss: 0.000486, epsilon: 0.010000, episode:  445\n",
      "frames: 933000, reward: 17.300000, loss: 0.000535, epsilon: 0.010000, episode:  445\n",
      "frames: 934000, reward: 17.300000, loss: 0.000354, epsilon: 0.010000, episode:  445\n",
      "frames: 935000, reward: 17.900000, loss: 0.000448, epsilon: 0.010000, episode:  446\n",
      "frames: 936000, reward: 17.900000, loss: 0.000598, epsilon: 0.010000, episode:  446\n",
      "frames: 937000, reward: 17.500000, loss: 0.000219, epsilon: 0.010000, episode:  447\n",
      "frames: 938000, reward: 17.500000, loss: 0.001618, epsilon: 0.010000, episode:  447\n",
      "frames: 939000, reward: 17.500000, loss: 0.000422, epsilon: 0.010000, episode:  447\n",
      "frames: 940000, reward: 17.100000, loss: 0.000628, epsilon: 0.010000, episode:  448\n",
      "frames: 941000, reward: 17.200000, loss: 0.000494, epsilon: 0.010000, episode:  449\n",
      "frames: 942000, reward: 17.200000, loss: 0.001387, epsilon: 0.010000, episode:  449\n",
      "frames: 943000, reward: 17.800000, loss: 0.000359, epsilon: 0.010000, episode:  450\n",
      "frames: 944000, reward: 17.800000, loss: 0.000697, epsilon: 0.010000, episode:  450\n",
      "frames: 945000, reward: 18.000000, loss: 0.001519, epsilon: 0.010000, episode:  451\n",
      "frames: 946000, reward: 18.000000, loss: 0.000419, epsilon: 0.010000, episode:  451\n",
      "frames: 947000, reward: 17.800000, loss: 0.000991, epsilon: 0.010000, episode:  452\n",
      "frames: 948000, reward: 17.800000, loss: 0.000279, epsilon: 0.010000, episode:  452\n",
      "frames: 949000, reward: 17.800000, loss: 0.000232, epsilon: 0.010000, episode:  452\n",
      "frames: 950000, reward: 17.200000, loss: 0.001161, epsilon: 0.010000, episode:  453\n",
      "frames: 951000, reward: 17.200000, loss: 0.000284, epsilon: 0.010000, episode:  453\n",
      "frames: 952000, reward: 17.300000, loss: 0.000331, epsilon: 0.010000, episode:  454\n",
      "frames: 953000, reward: 17.300000, loss: 0.000699, epsilon: 0.010000, episode:  454\n",
      "frames: 954000, reward: 17.400000, loss: 0.000345, epsilon: 0.010000, episode:  455\n",
      "frames: 955000, reward: 17.400000, loss: 0.000230, epsilon: 0.010000, episode:  455\n",
      "frames: 956000, reward: 17.700000, loss: 0.000395, epsilon: 0.010000, episode:  456\n",
      "frames: 957000, reward: 17.700000, loss: 0.000160, epsilon: 0.010000, episode:  456\n",
      "frames: 958000, reward: 18.100000, loss: 0.000436, epsilon: 0.010000, episode:  457\n",
      "frames: 959000, reward: 18.100000, loss: 0.000245, epsilon: 0.010000, episode:  457\n",
      "frames: 960000, reward: 18.500000, loss: 0.000190, epsilon: 0.010000, episode:  458\n",
      "frames: 961000, reward: 18.500000, loss: 0.000406, epsilon: 0.010000, episode:  458\n",
      "frames: 962000, reward: 18.200000, loss: 0.000246, epsilon: 0.010000, episode:  459\n",
      "frames: 963000, reward: 18.200000, loss: 0.000270, epsilon: 0.010000, episode:  459\n",
      "frames: 964000, reward: 18.000000, loss: 0.000444, epsilon: 0.010000, episode:  460\n",
      "frames: 965000, reward: 18.100000, loss: 0.000229, epsilon: 0.010000, episode:  461\n",
      "frames: 966000, reward: 18.100000, loss: 0.000179, epsilon: 0.010000, episode:  461\n",
      "frames: 967000, reward: 18.100000, loss: 0.000200, epsilon: 0.010000, episode:  461\n",
      "frames: 968000, reward: 17.800000, loss: 0.000381, epsilon: 0.010000, episode:  462\n",
      "frames: 969000, reward: 18.600000, loss: 0.000528, epsilon: 0.010000, episode:  463\n",
      "frames: 970000, reward: 18.600000, loss: 0.000553, epsilon: 0.010000, episode:  463\n",
      "frames: 971000, reward: 18.600000, loss: 0.000612, epsilon: 0.010000, episode:  463\n",
      "frames: 972000, reward: 18.600000, loss: 0.000204, epsilon: 0.010000, episode:  464\n",
      "frames: 973000, reward: 18.500000, loss: 0.003617, epsilon: 0.010000, episode:  465\n",
      "frames: 974000, reward: 18.500000, loss: 0.000295, epsilon: 0.010000, episode:  465\n",
      "frames: 975000, reward: 18.600000, loss: 0.001166, epsilon: 0.010000, episode:  466\n",
      "frames: 976000, reward: 18.600000, loss: 0.000161, epsilon: 0.010000, episode:  466\n",
      "frames: 977000, reward: 18.800000, loss: 0.000304, epsilon: 0.010000, episode:  467\n",
      "frames: 978000, reward: 18.800000, loss: 0.000163, epsilon: 0.010000, episode:  467\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 979000, reward: 19.000000, loss: 0.000283, epsilon: 0.010000, episode:  468\n",
      "frames: 980000, reward: 19.000000, loss: 0.000291, epsilon: 0.010000, episode:  468\n",
      "frames: 981000, reward: 18.900000, loss: 0.000218, epsilon: 0.010000, episode:  469\n",
      "frames: 982000, reward: 19.000000, loss: 0.000595, epsilon: 0.010000, episode:  470\n",
      "frames: 983000, reward: 19.000000, loss: 0.000151, epsilon: 0.010000, episode:  470\n",
      "frames: 984000, reward: 18.900000, loss: 0.000151, epsilon: 0.010000, episode:  471\n",
      "frames: 985000, reward: 18.900000, loss: 0.000152, epsilon: 0.010000, episode:  471\n",
      "frames: 986000, reward: 19.600000, loss: 0.000155, epsilon: 0.010000, episode:  472\n",
      "frames: 987000, reward: 19.600000, loss: 0.000151, epsilon: 0.010000, episode:  472\n",
      "frames: 988000, reward: 19.500000, loss: 0.000253, epsilon: 0.010000, episode:  473\n",
      "frames: 989000, reward: 19.700000, loss: 0.000289, epsilon: 0.010000, episode:  474\n",
      "frames: 990000, reward: 19.700000, loss: 0.000222, epsilon: 0.010000, episode:  474\n",
      "frames: 991000, reward: 20.000000, loss: 0.000221, epsilon: 0.010000, episode:  475\n",
      "frames: 992000, reward: 20.000000, loss: 0.000139, epsilon: 0.010000, episode:  475\n",
      "frames: 993000, reward: 19.700000, loss: 0.000215, epsilon: 0.010000, episode:  476\n",
      "frames: 994000, reward: 19.700000, loss: 0.000375, epsilon: 0.010000, episode:  476\n",
      "frames: 995000, reward: 19.700000, loss: 0.000110, epsilon: 0.010000, episode:  477\n",
      "frames: 996000, reward: 19.700000, loss: 0.000891, epsilon: 0.010000, episode:  477\n",
      "frames: 997000, reward: 19.500000, loss: 0.000190, epsilon: 0.010000, episode:  478\n",
      "frames: 998000, reward: 19.500000, loss: 0.000228, epsilon: 0.010000, episode:  478\n",
      "frames: 999000, reward: 19.400000, loss: 0.000223, epsilon: 0.010000, episode:  479\n"
     ]
    }
   ],
   "source": [
    "# if __name__ == '__main__':\n",
    "    \n",
    "# Training DQN in PongNoFrameskip-v4 \n",
    "env = make_atari('PongNoFrameskip-v4')\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True)\n",
    "\n",
    "gamma = 0.99\n",
    "epsilon_max = 1\n",
    "epsilon_min = 0.01\n",
    "eps_decay = 30000\n",
    "frames = 1000000\n",
    "USE_CUDA = True\n",
    "learning_rate = 2e-4\n",
    "max_buff = 100000\n",
    "update_tar_interval = 1000\n",
    "batch_size = 32\n",
    "print_interval = 1000\n",
    "log_interval = 1000\n",
    "learning_start = 10000\n",
    "win_reward = 18     # Pong-v4\n",
    "win_break = True\n",
    "\n",
    "action_space = env.action_space\n",
    "action_dim = env.action_space.n\n",
    "state_dim = env.observation_space.shape[0]\n",
    "state_channel = env.observation_space.shape[2]\n",
    "agent = DDQNAgent(in_channels = state_channel, action_space= action_space, USE_CUDA = USE_CUDA, lr = learning_rate)\n",
    "\n",
    "frame = env.reset()\n",
    "\n",
    "episode_reward = 0\n",
    "all_rewards = []\n",
    "losses = []\n",
    "episode_num = 0\n",
    "is_win = False\n",
    "# tensorboard\n",
    "summary_writer = SummaryWriter(log_dir = \"DDQN\", comment= \"good_makeatari\")\n",
    "\n",
    "# e-greedy decay\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_min + (epsilon_max - epsilon_min) * math.exp(\n",
    "            -1. * frame_idx / eps_decay)\n",
    "# plt.plot([epsilon_by_frame(i) for i in range(10000)])\n",
    "\n",
    "for i in range(frames):\n",
    "    epsilon = epsilon_by_frame(i)\n",
    "    state_tensor = agent.observe(frame)\n",
    "    action = agent.act(state_tensor, epsilon)\n",
    "    \n",
    "    next_frame, reward, done, _ = env.step(action)\n",
    "    \n",
    "    episode_reward += reward\n",
    "    agent.memory_buffer.push(frame, action, reward, next_frame, done)\n",
    "    frame = next_frame\n",
    "    \n",
    "    loss = 0\n",
    "    if agent.memory_buffer.size() >= learning_start:\n",
    "        loss = agent.learn_from_experience(batch_size)\n",
    "        losses.append(loss)\n",
    "\n",
    "    if i % print_interval == 0:\n",
    "        print(\"frames: %5d, reward: %5f, loss: %4f, epsilon: %5f, episode: %4d\" % (i, np.mean(all_rewards[-10:]), loss, epsilon, episode_num))\n",
    "        summary_writer.add_scalar(\"Temporal Difference Loss\", loss, i)\n",
    "        summary_writer.add_scalar(\"Mean Reward\", np.mean(all_rewards[-10:]), i)\n",
    "        summary_writer.add_scalar(\"Epsilon\", epsilon, i)\n",
    "        \n",
    "    if i % update_tar_interval == 0:\n",
    "        agent.DQN_target.load_state_dict(agent.DQN.state_dict())\n",
    "    \n",
    "    if done:\n",
    "        \n",
    "        frame = env.reset()\n",
    "        \n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        episode_num += 1\n",
    "        avg_reward = float(np.mean(all_rewards[-100:]))\n",
    "\n",
    "summary_writer.close()\n",
    "torch.save(agent.DQN.state_dict(), \"trained model/DDQN_dict.pth.tar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwYAAAE/CAYAAAD8LNSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeXydZZ3//9cnSZtu6ZakpfvOUtmEsqkIWlEQEWYUBRVQcZhxhh/jqDOiouPX0e+4zIzLgCiI28yXtaggVECWsi9tofuapkvSptn3fbl+f9z3Se6cnC3JOTknyfvZx3n03Pt1TpL7vj7Xas45RERERERkfMtKdwJERERERCT9FBiIiIiIiIgCAxERERERUWAgIiIiIiIoMBARERERERQYiIiIiIgICgzGFDM7yczeMrNGM7sl3emR9DKzDWb2uXSnQ0Qyl5kdMrP3pTsdIpIZFBiMLf8CbHDO5TnnfpruxIQzsyvMbIeZNZnZK2a2OrAt18x+ZGbHzKzWzH5mZhMC208xs2fNrN7Miszsr8LO/Tl/fZOZPWFm8wPbZprZb82swn99K+zYd5jZG35Atc3M3pXCryEjmdmpZvakmVWZ2YDJTcxsqZmt9382x83sdjPLSeC8vzYzZ2YrU5NyEZGxycw+5j8rW8xsQ4TtZ5rZZn/7ZjM7M7DNzOz7Zlbtv35gZjaENChwHGcUGIwtS4Cd0TaaWfYIpiX82quA/wf8HTAT+BPwaCBzeSuwBjgVOBE4C7jNPzYHeAR4DJgN3AT8r5md6G+/CPi/wJX+9oPAfYHL/wiYAiwFzgWuM7PP+MfOBh4Ffuin6wfAn8xs1hA+o5nZiP9NJZJBT0An8CBwY5TtPwMqgHnAmcBFwN/HSde7gBVJSJuIyJAk6f6YLjXAj4HvhW8ws4l4z8X/BWYBvwUe8deD95y8CjgDOB34EPC3I5BmGe2cc3qNgRfwLNANtAFNeJnr3wB3AuuBZuB9wOXAW0ADUAJ8K3COpYADPuNvq8XLyJ8DbAPqgNvDrvtZYLe/75PAkijpuxl4PLCcBbQCa/3lTcDVge2fAEr896f6n8kC258C/s1//x/AHYFt8/3PscJfrgLOCWz/GvCi//5DwM6wtO4Dbkzwe98AfBd42f88K4EZwD1AGXAU+A6Q7e9/GDjbf/8pP52r/eXPAX/0358LvOp/52XA7cDEwHUd8A/AfuCgv+4SYA9Q7+//PPC5Qf4erfRuCwPW7wY+GFj+IfCLGOfJ8X/PTvfTujLdfyN66aXXwBdwCHif/z4XLyN6zH/9GMj1txXgFc7U4WVYXwSy/G1f8e91jcDe0H09gWvfChzwj9sF/FUgHXXAqYF9C/177Bx/+UPAFn+/V4DTwz7TV/CeW+3+/Sjitfz9s4H/9J8VB/GeVw7I8bdHvadH+EyxvsOLgVLgS3gFLWXAZxL4nj6H1xoguO79flqCz8UjwKX++1eAmwLbbgRei3L+iD9b4H+AHv97b8JrlbDU/25u8j9fGfClwLmyAt91NV6B02x/2yS8QKbav9ZGYG66/wb06v9SjcEY4Zx7L94f883OuWnOuX3+pk/gZVzzgJfwAoTr8UrHLwc+b2ZXhZ3uPGAV8HG8m9rX8YKKtwEf80vo8Y/7GvDXeDftF+lfUh9k/it8+dQY2xea2Yyw9cHtsY4lsJ0I26MdG749Edfh3STz8DL+vwW68DLZb8e7gYfa+j+P93AAeDdQjFf6Hlp+3n/fDfwT3g37AmAtA0vor8L7Wa02swLgYbxalgK8m/I7ez+Q2WIzqzOzxYP4XEE/Aa4xsylmtgC4DHgixv7/BLzgnNs2xOuJyMj7OnA+Xq3gGXgFFLf5276El6ktBObi3fudmZ2El5E+xzmXB3wAL2OeiAPAhXgZ7/+DVxM8zznXDvweuDaw78eA551zFWZ2FvArvBLwfOAXeDXQuYH9r8V7xs10znVFu5a/79/g3dPOxKutDn8mxrqnh4v1HQKc4KdhAV5m/Y6h1FDjPY+3OT/H7dvmrw9t3xrYtjWwLVzEn61z7jq8YOMKP1/xg8Ax78HLJ7wfuDXQ3OgWvO/vIrxCulrgDn/bDXiffRHez+3v8IIOySTpjkz0St4Lr/T6c4Hl3wC/i3PMj4Ef+e+X4pUELAhsrwY+Hlh+GPiC//7PBErW8UoKWohQawCcjBeUXAxMBL6BVxLxVX/7d/BK3Qvxbpyv+2mZB0zAy0D/i//+/UAH8KR/7Fq8kp7Tgcl4D4ke4Fp/+//iPWTy8G7sB4B2f1s+XsnFtf65b/CPjVoaHuE7/3ZgeS5eCdXkwLprgef89zcCj/rvd+M9XO73lw8DZ0W5zheAPwSWHfDewPL1BEqD8IKbUpJXY3AKsBnv4ej83y2Lco5FQBEwI5BW1RjopVcGvuhfY3CA/jWDHwAO+e+/jdd0ZWXY8SvxSr/fB0wYZlq2AFf6798HFAe2vQxc77+/E7/GOLB9L3BR4DN9dhDXehb428C29/n3rZx49/QI5431HV6MlxHOCWyvAM6Pk9ZINQbfCD07Auv+H34rALzCpZMD21b5n2nAfTvazzb898NfXuqfJ3juHwD3+O93E6gxwnuGd/rf5WcJq93RK/NeqjEY+0qCC2Z2npk9Z2aVZlaPF7EXhB1THnjfGmF5mv9+CfATvyQ6VAVpeCUh/Tjn9uBlum/Hq3oswKvOLfV3+S5e05MteDeOP+LdTCqcc514JRCXA8fxSjceDB3rnHsG+Fe8oOUw3o2sMXDuW/x078e7+d0XOLYar2/CF/3PeSnwdODYRAS/4yV4AUZZ4Hv5BTDH3/48cKGZnYBXff0A8E4zW4pXkrIFwMxONLPH/I6+DXh9KMJ/TsHrzg8uO++OXEIS+P0mnsQLrqb66ZgFfD/KIT/GC5bqk3F9ERkx8/HuoSGH/XXgNR8sAp4ys2IzuxXAOVeEV3DxLaDCzO4PDv4Qi5ldb2ZbAvfKU+m7zz0LTPafWUvwSuD/4G9bAnwpdJx/7KJAWmHgsy/WteaH7T+Ye3q4WN8hQLXzajBCWuh7pg5GEzA9bN10vGdfpO3TgSb/2RAu4s82juB3FPyMS4A/BL6r3XhByly8pklPAvebN9DIDywwyIhkBgUGY1/4TeBevM62i5xzM4CfE7mpTiJK8EpZZgZek51zr0RMiHPrnHOnOufy8TLyS/DaGOKca3XO3eycW+CcW45XU7HZOdftb9/mnLvIOZfvnPsAsBx4I3DuO5xzq5xzc/AChBxgh7+txjn3SefcCc65t+H93gePfd45d45zbjZes6CTgtsTEPyOS/BKlwoC38l0/7qhh2gLXrDygnOuES/YuQl4yTnX45/nTrz+Aqucc9PxqnbDf07B65bhPRgBryN0cHmYZvvnut051+4HU78GPhhl/7XAD/2g5ri/7lUz+0SS0iMiqXEM774csthfh3Ou0Tn3Jf/+fAXwRTNb62+71zn3Lv9YR/RCg15+Zv9uvGZI+c65mXj3bPPP2YNXAHQtXpPYx/z7JXj32e+GPXumOOeCTVldotfCu38uDBwbvHfGvKdHEPU7TLKdwOlhIw2dTt8AJDvxmjKFnEGUwUli/WwZmIcICX5Hwc9YAlwW9rOZ5Jw76pzrdM79H+fcauAdeP1Erk/s48pIUWAw/uQBNc65NjM7F++GO1Q/B75qZm8DMLMZZnZ1tJ3N7GwzyzazQrwSlz/5NQmY2QIzm++P7HM+XjXpvwaOPd3MJvlt3L+MVz35G3/bJPOG2zS/Df1dwE+cc7X+9hVmlu9f+zK8TPh3Aud+u5lNMLPpeB2ZS51zTw7lC3HOleF1jP5PM5tuZln+9S8K7PY83gMq1J9gQ9gyeD+nBqDJzE4GPh/n0o8DbzOzv/ZH4bgFr0lWQvzvbhJeM6/Qd5rrf6ZQh7zPm1mOmc3Eq/3ZGuV0J+I9hM70X+A9bP4QZX8RyQz3AbeZWaHfb+mbeE0xMbMPmdlKPyPagFcK3G3e/Dnv9e8XbXi1s93+MRdbhOGPfVPxMp2V/r6fYWDfrnvx+rp90n8fcjfwd35tgpnZVDO73MzyhnitB4F/9J9DM/E6LgMJ39ODon6Hg+U/sybhFXRl+fflUAn7Brzv+Rbzhvu+2V//rP//7/Ay+Av8Gpwv4T8zI1wn4s/W31yOVxAX7hv+8/hteAOWPOCv/znwXT8Yw/8ervTfv8fMTjNvhMQGvFYB3RHOLWmkwGD8+Xvg22bWiHfDenCoJ3LO/QGvZOh+85q77MDrwBXNT/Da8+/1//+bwLYVeE2ImvE6et3qnHsqsP06vFKdCrwS6Uuc10ENvJEO7sWrOn0DbzSfbwSOPRvYjlfF+u/AJ51zwZKTf8Hro1CCF3D0zpFgZheaWVPML2Kg6/Ey2LvwOl6t888b8jxexv+FKMsAX8YL2hrxHoIPEIOfeb8ab1i7arz2pC8HPsdi8+Z4iNb5eAneAz30vbTi/ZxC/hqvmVUlXpVzF14H49D5m8zsQj8tFc6546GXv0uVc06dzEQy23fwRojbhnfPfJO+QpRVeM0sm/DusT9zzm3AG4Xne3j30ON4TWy+5h+zyN93AOfcLryRgF7Fy3yeRuCe5e/zOt4zYT5en7bQ+k14z4/b8e6xRcCno32oBK51N17mfxtek9b1ePe4UKY13j09KNZ3OFjX4d2L78TrON3qpxXnXAdeE9vr8Z6nnwWu8teDX/jmp2EHXuHRL6JcJ9rPFrxn5m1+06AvB455Hu97fwb4j8Dz+id4rRKe8vMZr+ENkgFeYdU6vKBgt3+OIQVNkjoWubmZiIiIyNCZ2S+Bh4ZaA5sufs3yz51zS+LuPM6Y1x/uIF5H867Ye8toNJon/hAREZEM5ZyLNqRnRjGzyXjDbz6F10n2X1HTRxmn1JRIRERExjPDm9ugFq8p0W68prYi446aEomIiIiIiGoMREREREREgYGIiIiIiJBhnY8LCgrc0qVL050MEZGMtHnz5irnXGG605FOek6IiESWjGdERgUGS5cuZdOmTelOhohIRjKzw+lOQ7rpOSEiElkynhFqSiQiIiIiIgoMRESkj5ldamZ7zazIzG6NsD3XzB7wt7/uT3iEmeWb2XP+LNi3B/bPM7MtgVeVmf3Y3/ZpM6sMbBsV496LiIxVGdWUSERE0sfMsoE7gEuAUmCjmT3qnNsV2O1GoNY5t9LMrgG+D3wcaAO+AZzqvwBwzjUCZwausRn4feB8Dzjnbk7RRxIRkUFQjYGIiIScCxQ554qdcx3A/cCVYftcCfzWf78OWGtm5pxrds69hBcgRGRmq4A5wIvJT7qIiAzXsAMDM1vkVx/vNrOdZvaP/vrZZvYXM9vv/z9r+MkVEZEUWgCUBJZL/XUR93HOdQH1QH6C578Wr4YgOLPmR8xsm5mtM7NFkQ4ys5vMbJOZbaqsrEzwUiIiMljJqDHoAr7knDsFOB/4BzNbDdwKPOOcWwU84y+LiEjmsgjr3BD2ieYa4L7A8p+Apc6504Gn6auJ6H9y5+5yzq1xzq0pLBzXo7WKiKTUsAMD51yZc+5N/30jsBuvRClY3fxb4KrhXktERFKqFAiW2i8EjkXbx8xygBlATbwTm9kZQI5zbnNonXOu2jnX7i/eDZw99KSLiMhwJbWPgT86xduB14G5zrky8IIHvHalIiKSuTYCq8xsmZlNxCvhfzRsn0eBG/z3HwWeDWsaFM219K8twMzmBRY/jFewJCIiaZK0UYnMbBrwMPAF51yDWaTa5ojH3QTcBLB48eJkJUdERAbJOddlZjcDTwLZwK+cczvN7NvAJufco8A9wP+YWRFeTcE1oePN7BAwHZhoZlcB7w+MaPQx4INhl7zFzD6M1yS1Bvh0yj6ciIjEZYkV9MQ5idkE4DHgSefcf/nr9gIXO+fK/FKhDc65k2KdZ82aNU4zWorIYLV3dfPCviouWJHPtNwctpfWMzU3m+WF09KdtKQys83OuTXpTkc66TkhIhJZMp4Rw64xMK9q4B5gdygo8IWqm7/n///IcK8lIhLJPz+0jUe3HqNgWi5Xnjmfe146SHaW8cQ/XsiquXnpTp6MIi/ur+TCVergLCLjUzL6GLwTuA54b2D2yg/iBQSXmNl+vMlyvpeEa4mI0NzeRUWDN1x+a0c3T+48DkBVUzv3vHSQMxbNpLvH8YsXitOZTABe2FfJL18s5nt/3sNzeyrSnRyJo6SmNd1JEBFJm2HXGPgT2kTrULB2uOcXkdEr1FQx0T5Hifrrn73C3vJGNt32Pi7/6Yu0d/Xwq0+vYcrEHM5cNJNJE7L5xh93cO8bR+js7mH1vOn87UUrkpqGRDy4sYR/eXhb7/IvXjjAf33sDP7q7QtHPC0iIiLxJK3zsYhISElNC9uP1vOrlw6ycNZkfnzN25N27q0ldewtbwTgQz99ifKGdm5693IuOnEO2Vl9Aci/XHoSFY1tvFxUzSNbjnH2klmYwawpE0ek70FNcwdf+8N2zls2m3etLGDtKXP53auHOHvx7JRfW0REZCgUGIhI0n35oa28ftAb2n7T4dqkBQad3T08sMmbmHdabg7TJuVw91VruGT13AH75k2awC+uW0NrRzcX/uA5bvvjDvYc9wKK+/7mfC5YkehkvYnr6OrhkS1HufLMBTy3p4KuHsdtl6/mtIUzAPjeR05P+jVFRESSRYGBiCRVVVM7rx+s4eQT8jh1wQzWbS6lpaOLKROHf7v57uO7uff1I5y/fDa/++x55GQZWVmxmylNnpjNp85fzI+f3g/AhGzja3/YzjNfvCjusYN1/8YjfPORnRyqbmZbaT1zp+dy6oLpSb2GiIhIqigwEBkHOrt7uPrnr1Ld3M7N71nJsoJpnL5wBsWVzayeP/SMa2tHNy8VVVFW30pVUwfvWJHPvz6yE4AffvQMDlU3s25zKUdqWjj5hOFlkOtbOrnvjSMA/PMHTmJiTuJjJ9xwwVLK6to46YQ8CvJyueW+t/jVywf53IXLh5WmcE/s8DpB3/HcAQC+9sGTk96/QkREJFUUGIiMA0/tLGdLSR0AX3l4OwA5WUZXj+OeG9aw9hSvKU5ndw/P7C7n/OX5zJwyMeK52ru6+fofdjB1YjZ7jjf2NhkC+OkzXqn8Vy49ubf5DMChquZhBwZ3v1hMe1cPf/7HCzll3uDONWvqRL7/Ua8ZT0dXD6cvnMF3Ht9NYV4uV565YFjpCilvaOPV4mo+886lFEzLpa2zm+svWJqUc4uIiIwEBQYiY1xZfSu3/XE7ywqmsv6WCzlY1czj24+xYW8lO4818F9/2cfywmm0dnTzb4/t4tXiaqZMzOa0BTP43IXLB7Tff+NgDes2l/Yuf/2Dp/Cek+ew53gDN9/7FgCfeedSAJYVTmXKxGxu/f12dpU18o4V+UyfNGHQtRS7yxq468Virjhj/qCDgnATc7J4+PPv4CN3vsJPnt4/rMCgvasbw5iYk8UjW47iHHzq/CWsGGMTq4mIyPigwEBkjHtxXxW1LZ38z43nMXliNqvnT2f1/On88wdO5s4NB/j+E3t4z39sACBvUg6feedSmtu7eGFfFTff+yavfXUts6b21R68VFTFhGzj8xevZO70XD553hIAVs6ZxhkLZ1LV1M6kCdmA10H4158+h+89sYefPrO/t0Zh73cuJTcnO27aXz1QzTO7y1n3ZikzJk/gX69YnZTvZEJ2Fh8+Yz7feXw3rxyo4oLl+YNu8tPY1sl7/mMDp8ybzntPnsP3/ryHc5fNVlAgIiKjlgIDkQRtL63nxBOmJZShTYfuHsd/P7ufupZOzl02mw+eNg+APccbmTQhK2JJ+/vfNpfvP7GHMxfN5MxFM7nhHUtZVjDVP66BS3/8Ive+cYR/eM9KGts6+cidr7CvvInzl8/mi5ecOOB8i2ZPYdHsKf3Wnbc8n99//h38ze828fRub4Kvf35oG//5sTOYkB25n8DTu8q55f63aOno7lv3xYsomJY7tC8ngnetKgDgE3e/zswpE/jva9/OrCkT2bC3gr+9aEXEtDnn+M+n9lFW38bz+yqoaurgxf1VvLi/ihmTJ/DVy05OWvokPRwu3UkQEUkbBQYiCSirb+WK21/ixLnTuOndK7j8tHlMnpjN9tJ6CvImMm/G5HQnkS0ldb0j7/zmlUP8/FNncemp89hb3sBJc/P6jfEfsqJwGk9/8SKWF0wdMELPySdM5+KTCrlzwwEOVjVTUtPCvvImwOtDMBhmxi+uW0N5QxuPbDnG95/Yw8wpE/jGh1aTk2X9Suu7exzffmwXLR3dfHzNIs5ZNpuFsyazck5yS+JPmpvHv135Nioa2/nvZ4v49/V7qG3poKy+jS0lddzxybMGBIH7ypu4/bki8nJzaGzv4uKTCuno6uH6C5Zw6anzkpo+GXkVjW10dvUAsGFvBSvnTGPhrClxjhIRGTsUGIgk4K0jXsfdfeVNfPmhrbR2dnP12Qu54vaXyJuUw/ZvfYDO7h5KalpYVjA1LSPRbPU7Fz968zu57Y87+P/ue4vTFx5k8+FaPrYm+ky7sTLcX7n0ZC77yYu9fQrOWzabn3/q7H5NixKVnWXMnzmZz1+8grL6Vn736mF+9+phPn/xCr5y6clUNbVTMC2Xt47UcqSmhR985HQ+ds6iQV8nUWbGdX7n4OmTJvDd9bsBuPFdy7jnpYN869FdfOLcxTy16zivF9dwyrw8lvq1KY/c/E7eOlLH+1bPZcbkCSlLo4ysNw/X0dTeBcCxujaO1bXxifMWpzlVIiIjR4GBSAK2ltQxMTuL5//lYq7475fZcqSO6ZO8P5/Gti5+8vR+/rL7ODuONvCxNQv5wUfPGLG0vVJUxfodZfzva0eYMXkCpy+cyd3Xr+HHT+/jvje8ycCuXjO0DPYp86bzk2vOpKiiiZNPmM57Ti5MynwE/7h2FY9tK6OmuYM7NxwgNyert7Yj5G0jOP7/je9aRmtnNwtmTuYjZ3tB1D0vHewdHnXx7Cm8ccgbfWlOXi7LCqaOyOzJIiIiI0mBgUgCXjtYw+r505k3YzInnTCNh98s5eE3S5mTl8ui2VP40dP7ADhr8Uwe3FRKwbRc/ubC5UMqWY+msa2TqqaO3j4AAHuPN/KJX77eu3ztuV7p5tzpk/j3vz6dj569kB1HGzhn6ewhXzdZw3kG5U/L5ZkvXsS9bxzhh0/uHRAUQOyajGTLyjJuWbuqd/m2y09h7SlzeGpnOTe+axmLZk/h3teP8LMNRfzT+07U3ATjRLdTfwMRGV8UGIjEcd8bR9haUtfbrv6SU+byclE1152/hJvfu5JJE7I569/+wmWnnsCPPn4mH//Fq/xswwHufeMIT/3Tu5mTN2nYaWho6+SqO16muLKZJ7/wbk46IY/128v4yrptzJwygY+tWcT1FywZ0B767CWzOXvJ0IOCVJo1dSJ/f/EKzl+eT2d3D2ctnsVDm0v4+h92AKS1k7eZ8Y4VBbxjRUHvuk+ct1jNSsaQI9UtLM6P3X/gwY0lXHf+khFKkYhI+ikwEImhsrGdr/7emxDsyjPnA3D9BUv5yNkLyZvU17Z8823vY8rEHCZkZ3H/TRfw7J5y/u5/3+ThzUf5/MUrhp2OT9z9GsWVzQD88sVifnj1GdzxXBFTc3N48G8viJvByVRmxtlLZvUuf/K8JZyxcCYd3T1pTJWMBy8VVfGJ/L5Az5sAUDUEIjK+RR4rUEQA2OS3K//tZ89l/kxv5KGsLOsXFADMnDKRiTnen9PEnCwuPXUe5y6dzUObSnDDbI5QUtPCjqMN3LJ2Fe89eQ4PbS5l3eZSdpc18PFzFo3aoCCaUxfM4KzFs+LvKJJEu441AHDHcwfSnBIRkfRRYCASw8ZDtUyakMUFy/MHfezVaxZSXNXMKd98gsPVzew53jCkNLxcVAXAFafPY7U/F8GXH9pKj4M1S5WBFkmm1s5uDlY1pzsZIiJpocBAJIb9FY2cNDevtzZgMC4/fR5nLJxBW2cPF/1wA5f++EWO1bUO+jx7yxuZOjGblXOm8Y4VfQHKp85fPKSARUQ80QKAVw9UA95kZzuO1rOvvHEkkyUikjYKDERiqGxsp3CInYenTMzhkZvfxWfeubR33cP+fACDUd3UQUFertchdmUBm257H4e+dznfueo0cqLMHCwi8YVq40IidW0pqWmhq1t9D0RkfFCuQiSGqqYOCvOGN+ToFWfM733/0OZSenoGl8moae4gPzDsacG03GGlR0S82oBw4cOTGsZzeytGKkkiImmnwEAkiu4eR01zO4XDzIiftXgWv//7d/DDj57OkZoWNh+pHdTxVU3t5CsYEEmbSEGEiMhYpMBAJIqa5g56HBTkDT9TftbiWZzv9wcYbMfG6rAaAxEZPmPgJHXBNaFgwDAaWrtGKFUiIumleQxEoqhqagcYdo1BSP40L3Nf3dSR8DE9Pc5rSjRNgYFIMsWrBXhwYwnN7V38ZXc55Y1tXLBCHf1FZOxLSo2Bmf3KzCrMbEdg3bfM7KiZbfFfH0zGtURGyh3PFQHJqTEArzPy5AnZ1DS3J3xMQ1sn3T2O2VPVlEgk1baW1PW+73FQXNVMTXMHL+yrTGOqRERGTrKaEv0GuDTC+h855870X+uTdC2RlCuqaOKxbWUArCiclrTzzp46cVA1BhWNXhBRoBoDkaQyjD++dbR32eHYcaw+4r7qYSAi40VSAgPn3AtATTLOJZIJHtpUQnaW8cbX1zI7ie37C6ZNpKrZCww2H66loqEt5v6h8dRPWzAjaWkQicXMLjWzvWZWZGa3Rtiea2YP+NtfN7Ol/vp8M3vOzJrM7PawYzb45wzVIM+Jda5U2RAYYcjhaO3s7v/Z/F4GtS39g/dhTl4uIjJqpLrz8c1mts1vahRxilYzu8nMNpnZpspKVddK+j23p4JfvFDMe0+ew5whzmEQzeypE6lpbqezu4eP3PkK5/7fZ2jv6h6wX0tHF60d3fx5RxnLC6ayPIm1FiLRmFk2cAdwGbAauNbMVoftdiNQ65xbCfwI+L6/vg34BvDlKKf/ZKAGOZRDj3aulDhW5wXijW2d/Pv6PVH3e2LHcXoUDYjIOJTKwOBOYAVwJlAG/GeknZxzdznn1uFBd8MAACAASURBVDjn1hQWFqYwOSLxNbR18pnfbATg42sWJf38+dNyqW7qYO/xvplUf/L0fq6561Vq/ZqE7h7H1T9/lVO++QSvFddw7bmLk54OkSjOBYqcc8XOuQ7gfuDKsH2uBH7rv18HrDUzc841O+dewgsQEhXxXENPfmJaO7ppau8/0lBwlCKHY+exhqTWFoqIjAYpCwycc+XOuW7nXA9wN94DRySjPb/Xq7X6yqUns/aUOUk//9zpuVQ0tvPTZ/b3rvvZhgO8VlzD+h1l7C5r4Lz/+ww7jzUwd3out6xdxWfftSzp6RCJYgFQElgu9ddF3Mc51wXUA4kM2fNrvxnRNwKZ/6GeK2ka2jp734dGKgpVFiwvmDqSSRERSbuUBQZmNi+w+FfAjmj7imSKNw7WkDcph5vevZxUFFy+56Q5dPc4ntpVzpy8XL51RV8rjZ89d4CP/+JVwPGlS07k1VvX8sVLTiQ7K+UFqCIhkX7ZwtvUJLJPuE86504DLvRf1w3mXKlqcmoGj20t67uwWg+JyDiXlHkMzOw+4GKgwMxKgX8FLjazM/Fu8oeAv03GtURSqaKxjfkzJqcsM372klmsnDONWVMm8PNPnU3+tFyuOGM+j28v46FNpayYM43vXnUqi2ZPScn1ReIoBYJt6BYCx6LsU2pmOcAM4gw+4Zw76v/faGb34tUg/y7Rcznn7gLuAlizZs2Qs+/bj/qjDvl/3kdrW3u39ThHZ3ePfz1v3abDg5ulXERktEtKYOCcuzbC6nuScW6RkVTV1EFBXuraFZsZ62+5kOws6w0+8qflcv0FS7n+gqUpu65IgjYCq8xsGXAUuAb4RNg+jwI3AK8CHwWedS56Wbuf4Z/pnKsyswnAh4Cnh3Ku4brvjSP8+1+fxmvFXuxR29LXjOh4fRtvHvHmMdhV1pCqJIiIZLRUj0okMqpUN7WTn+LJxCbmZKl5kGQkv53/zcCTwG7gQefcTjP7tpl92N/tHiDfzIqALwK9Q5qa2SHgv4BPm1mpP6JRLvCkmW0DtuAFHHfHO1cq/eTpfb3vn9p1HECjEImIkKQaA5Gxoqqpg4JpmmVYxi9/Msr1Yeu+GXjfBlwd5dilUU57dpT9o55rpFT6kwj+bMOBdCZDRCQjqMZAxNfW6Q1hmK9ZhkXGjZ3HGnjlQFW6kyEikhEUGIj4qpq8ksNC1RiIjGkHKpv7Le88qj4FIiKgwECkV1WTN8GYagxExpe95Y3xdxIRGQcUGIj4Dld7pYgLZ2moUBERERl/FBiI+IormzGDJfkKDERERGT8UWAg4jtY1czCWZOZNCE73UkRERERGXEKDER8xVVNLCuYlu5kiEgKVfuDDAxWcWVTklMiIpJ5FBiI+I5Ut7BktpoRiYxl63ccH7AukekGXzlQnfzEiIhkGAUGIkBrRzcNbV2cMGNSupMiIqmkGY5FRKJSYCACHG9oA2DudAUGIuPNU7vK050EEZGMoMBABDhe7wUGJygwEBl36ls7050EEZGMoMBABCj3awxOmKFZj0XGMjUkEhGJToGBCGpKJCIiIqLAQAQ4WttK3qQc8iZNSHdSRERERNJCgYEIsL+ikZVzNIeByFinQYlERKJTYCACFFU0s7JQgYHIWOcUGYiIRKXAQMa9upYOqpraWTVXgYHIWPfC/qp0J0FEJGMpMJBx71B1CwDLChQYiIx1JTUt6U6CiEjGUmAg417vUKUakUhkzNtf0ZTuJIiIZKykBAZm9iszqzCzHYF1s83sL2a23/9/VjKuJZJsFY3tAMydrjkMREREZPxKVo3Bb4BLw9bdCjzjnFsFPOMvi2ScioY2sgzypykwEBERkfErKYGBc+4FoCZs9ZXAb/33vwWuSsa1RJKtvKGNwrxcsrMs3UkRERERSZtU9jGY65wrA/D/n5PCa4kMWXlDO3Py1L9ARERExre0dz42s5vMbJOZbaqsrEx3cmQcKm9oU/8CERERGfdSGRiUm9k8AP//ikg7Oefucs6tcc6tKSwsTGFyRCKrbGxnjkYkEhERkXEulYHBo8AN/vsbgEdSeC2RIeno6qG6uYO5akokIiIi41yyhiu9D3gVOMnMSs3sRuB7wCVmth+4xF8WySiVTRqqVERERAQgJxkncc5dG2XT2mScXyRVQpObzVFgICIiIuNc2jsfi6RTRSgwUFMiEQDM7FIz22tmRWY2YP4ZM8s1swf87a+b2VJ/fb6ZPWdmTWZ2e2D/KWb2uJntMbOdZva9wLZPm1mlmW3xX58bic8oIiKRKTCQca28IdSUSIGBiJllA3cAlwGrgWvNbHXYbjcCtc65lcCPgO/769uAbwBfjnDq/3DOnQy8HXinmV0W2PaAc+5M//XLJH6cpOpxLt1JEBFJOQUGMq6V1bcxIduYPXViupMikgnOBYqcc8XOuQ7gfrzJKoOCk1euA9aamTnnmp1zL+EFCL2ccy3Ouef89x3Am8DCVH6IVHhgY0m6kyAiknIKDGRcO1rXyvyZkzXrsYhnARDMAZf66yLu45zrAuqB/ERObmYzgSuAZwKrP2Jm28xsnZktinKc5rsRERkBCgxkXCutbWHhrMnpToZIpogUIYe3oUlkn4EnNssB7gN+6pwr9lf/CVjqnDsdeJq+moj+J9d8NyIiI0KBgYxrpbWtLJw5Jd3JEMkUpUCw1H4hcCzaPn5mfwZQk8C57wL2O+d+HFrhnKt2zrX7i3cDZw8x3SIikgQKDGTcauvsprKxnQWqMRAJ2QisMrNlZjYRuAZvssqg4OSVHwWedS52z1wz+w5eAPGFsPXzAosfBnYPI+0iIjJMSZnHQGQ0Kq1tBVBTIhGfc67LzG4GngSygV8553aa2beBTc65R4F7gP8xsyK8moJrQseb2SFgOjDRzK4C3g80AF8H9gBvmhnA7f4IRLeY2YeBLv9cnx6RDyoiIhEpMJBx60BlEwArCqelOSUimcM5tx5YH7bum4H3bcDVUY5dGuW0EXv3O+e+Cnx1SAkVEZGkU1MiGbeKKvzAYI4CAxEREREFBjJuHaho4oTpk5iWq4ozEREREQUGMm4dqGxipWoLRCQBdS2d6U6CiEjKKTCQcck5x4HKZlYUTk13UkRkFGjv6kl3EkREUk6BgYxL5Q3tNLV3qcZARBISZ0RWEZExQYGBjEu9HY81IpGIJEBhgYiMBwoMZFwqqmgEUI2BiCSkprljUPtvOpTIZNAiIplFgYGMSwcqm8mblENhXm66kyIiY8zWkjr2lTelOxkiIoOmwEDGpaKKJlYUTsOfhVVEJGl2HmtIdxJERIZEgYGMSxqqVERERKQ/BQYy7jS0dVLR2M5yDVUqIilU3dSe7iSIiAyKAgMZd47WtgKwZLYCAxFJnSd3lqc7CSIig6LAQMadY3VeYDB/5qQ0p0RExqrObk2IJiKjT8oDAzM7ZGbbzWyLmW1K9fVE4gkFBgtmTk5zSkRkrFq3uTTdSRARGbScEbrOe5xzVSN0LZGYjta1MTE7i4JpGqpURERkvOjpcWRlaTTCWNSUSMadY3WtzJs5STcHEUm6Z/f09SuobRncpGgikloPbCpJdxIy3kgEBg54ysw2m9lNI3A9kZhKalvUjEhEUuJ4fd9IROu3l6UxJSISzrl0pyDzjURg8E7n3FnAZcA/mNm7gxvN7CYz22RmmyorK0cgOTKeOecormxmWYFGJBIREZHkae3oHtTAA3/ZlXkjl6U8MHDOHfP/rwD+AJwbtv0u59wa59yawsLCVCdHxrma5g7qWztZXqjJzURERCR5Nh+upayuLeH9Kxszb66TlAYGZjbVzPJC74H3AztSeU2RWA5WNQOwXDUGIiIiIv2kusZgLvCSmW0F3gAed849keJrikRVXOkHBpr1WESSoLiyKd1JEBFJmpQOV+qcKwbOSOU1RAZjX3kjuTlZLJw1Jd1JEZEx4LXiGjVNFJExQ8OVyriyr6KJVXOnka2hSkVERET6UWAg48q+442cOCcv3ckQERGRDFPR2MbrxdXpTkZaKTCQcaO+tZPjDW2smqvAQESSw+ENjL6vvDHNKRGR4erqdrR0dqc7GWmlwEDGjaIK78F94ly1BxaR5Np0qDbdSRCRQWrt6Ka9a3wHAuEUGMi4sassFBioxkBEksOI3l/J1JVJJKPtPFbP4eqWdCcjoygwkHHhteJqvvFHbwqNBTMnpzk1IpnLzC41s71mVmRmt0bYnmtmD/jbXzezpf76fDN7zsyazOz2sGPONrPt/jE/NfOyzGY228z+Ymb7/f9njcRnHCnOpTsFIuOTS8IfX0Nb5+Cvy+j/o1dgIOPCnrIGAM5bNpssjUgkEpGZZQN3AJcBq4FrzWx12G43ArXOuZXAj4Dv++vbgG8AX45w6juBm4BV/utSf/2twDPOuVXAM/5yRqtoaKO+JXKGobGtkw17K0Y4RSIS7qFNpVG3PbbtWMxjQ7MRP7a1bEjXHu01hQoMZFyoae4gy+Devzk/3UkRyWTnAkXOuWLnXAdwP3Bl2D5XAr/1368D1pqZOeeanXMv4QUIvcxsHjDdOfeq84rxfgdcFeFcvw2sz1gHq5qpaGyLuK3HQXNH1winSGR8eXhz9Ex/SFdP9JL7uiiBfcjTu8oHnaaxRIGBjAuVTe3Mnpqr+QtEYlsAlASWS/11EfdxznUB9UB+nHMGn+TBc851zpX55yoD5gw55Wn07J7xnZEQGUntXT3pTkKv3WUNHK1rTXj/0VCjqMBAxoXKxg4Kpk1MdzJEMl2kyDm86C2RfYaz/8ATmN1kZpvMbFNlZeVgDh0Rx+vb+y1Hamdc1dQ+YJ2IDF5XT8+AzPgjW46mJS31rZ20DqKW8Fhd/9rGTOyToMBAxoXKpnYK83LTnQyRTFcKLAosLwTCG+T27mNmOcAMoCbOORdGOWe539Qo1OQoYnGac+4u59wa59yawsLCBD9K+kRq3/zEjuNpSInI2NPa0c2bh/sPD9zcPvQhRzVIQH8KDGRcqGpsp3CaAgORODYCq8xsmZlNBK4BHg3b51HgBv/9R4FnXYwhQPwmQo1mdr4/GtH1wCMRznVDYP2o4XD88Mk9fcsOumO0bxaRzJGsjsLOOV7aXxV1++PbymiLMHFarOGO00WBgYx5zjmqmtopUI2BSEx+n4GbgSeB3cCDzrmdZvZtM/uwv9s9QL6ZFQFfJDCSkJkdAv4L+LSZlQZGNPo88EugCDgA/Nlf/z3gEjPbD1ziL2c0MxtQ+V/rd2Y08wY6EJGxqzpCs8AHNpbEHO1oNE2ilpPuBIikWmtnN+1dPcyeqj4GIvE459YD68PWfTPwvg24OsqxS6Os3wScGmF9NbB2GMlNqotPKmTD3sH1YQiW+K3bVMqe442ctmBGspMmMq5UNbXT1e04YcakYZ/rmd3lrD1lbhJS5fnzjuN86vwl/dZtLa1j+9F6Lj99XtKuky6qMZAxL1SaN2vKhDSnREQy2TlLZw/r+EwaLUVkNKtoaKesPvHRfmIpq488vHC45/ZU0NgWvSPx4ermqNu2ldYDw+uvMJQJ1VJBgYGMebV+1f6MyaoxEJE+eZP6V5qfsXDmgH1mTI5doBAcVSS8vXJwWb0ORIZvsDMa17XEbtrX0tHd731P2PmDTQNfLqqOn74I6w5VNfNihP4H4SMpDXVCtWRTYCBjXn2ragxEZKDwbn/BTP5Q5jwJHbH9aD2d3f1rD9ZtLqW6qT1jSgVFAHp6YneazTQPbvKmWUk0PFi/vW80sF3HGgZs/+Nbx3h6d+R5SHqc49O/fqN3+XB1M28d6RsNKdpEaOEFBK8cqOZITcuA/Zrbu3vvOccGMRdCqikwkDGv1i8xmDlFNQYikpge5zhx7jTmhbVxNuvfXCDaqCLO9d+vq7uH4qpmyhNs1iAyEhxwtG5gpjXTHKxqpqu7h+6eyCMJ/f7N+LMhbympA7w+B0EVDeFzC3h6nOOC5fn8eXsZLR1ddHY7Orv7/qgrGgc3N4lzffmR8JqPwfZtSiUFBjLmqY+BiESSHzaEcTCT7xzkZGX1qzmI1Iyh90GfQBnmdr8dskgmijZyzmvFfU1onoszc++bR2pjbh+M4F/Ui/sr6YoxDHB4/562zu6Iw4NC/wx9rOFKDVhROI32rh7KG4Y2QeETO45T09zee2cJzWfy8JtH/Wt4W1oDaS2JULswkhQYyJhXF+pjoMBARHynLZjBl95/4qCOef1gDUb/IOCNg97cbpEmNQu3tbRuUNcTGUkPb448e3BxZV+n27K6yDVeoUz4nrLGYabB+zsKz7A/siX6UKCR7K9oYn9505DTUdXUzmv+3/ZLRVURmw0lMgRpVVM7fw6b3LCjq4eOQCDzyJaj/Wo8nt+X3toDBQYy5tW1djJlYja5OdnpToqIZIBJE7xH3/RJEyKuD8nK6t8cKLxjYtC+8ugZoj9vLwvsN/TMiki6dHb3UO/Xvof3nwH4vV8CXhuns2886RjZq6PbselQ/8nbu7odLe3eCEWR+gdA32eOJ/y2sW5z/0KEWCMhpYMCAxnzals6mKX+BSLiW1E4DRhYKpmVZXznqr7pFj62ZlHM87R39Y1i0tLRHbVZQqg5o3Ow+XDsphaxJkkSSYXwjOsTYSXc4I3u9/pBr0lRrNqxSMcmS0NbJy8fiN9RunGQHfx7ehxHaloi/v1uPzr85n+7ygZ2es5kKQ8MzOxSM9trZkVmdmv8I0SSq7KxnULNeiwivqm53jCl4R2HDcgK5A5mT50YMbMQyki9XFRFZYwOiHuP969FcMRu0wxQ16JRi2TkRPp1TGT27mB/m9La4bWJf724/zCgrR2Rm+h09zgaWkNBdvTauz/5w37uKht8pj78tIkEBs/tyZyOw8mQ0sDAzLKBO4DLgNXAtWa2OpXXFAl3vL6NudMVGIiI58NnzAe8THrofVAo8/7BU+eRk50VcVsk4YFGpM6SRmIjqIgkQ2tHN90xOu0O1UOB5jAv7BvecKcHKvtPHPaHtyI30Qll2qONBObt0/dZ3zrS16fniR2pmyMg0QnUIg1QkMigBSMt1TUG5wJFzrli51wHcD9wZYqvKdJPeUMbc6cPf1p1ERndzlo8cAKzSP0GgtmOr3zgpITPH+w0GB5A7AyMod4SpURUJNle3F+ZUA1ALDuP9S81dzi6upOfoX25qH+A0X9Y4MSE5jkIV9UU/zt49UD8CczChQ8zGq0mI15NYSZJdWCwAAj+lEr9dSIjorWjm4a2LgUGItLPh8+YT96kHE4I3BvMf3pb4Cn+jpUFCZ/zWH3sSYoqGzWHgYw+ofH/YWAGN3xOgOE4XN3XJGmoGenObsfe4w0EBzstre3/dxnt3IeqvZqL0ObQGcJHIgsqi/I3X93UTld3D03tiXcsfmzbsUHP7JwKqQ4MIn39/T61md1kZpvMbFNl5dhqpyXpV+5PXKLAQESCD5+puTmcvnAmy/2OyEGDyZPYIHIwrxXXjKqSQxlbhjvrdqQs61DH91+/PXVNe970mxCF/tZe3O/lLTcdquFwTXPEY8JbW0X7Ow3PuEf7c/7zjuNUN3fErYUINovKlNGJUh0YlALBYR0WAv2GXHDO3eWcW+OcW1NYWJji5Mh40xcYqI+BiMBJc/Nibr9wVQGnzJsedXt4k4RgRiHR5hVvxhmZSCQVHts6jMx4kgPa8E72VU3tvHGwmpaOrqgdfp/d402uNtR2+Z09bsCM5CH3v3Ek7rmDwUJwtuTwY8yMN4/UxhzCON750ynVgcFGYJWZLTOzicA1wKMpvqZIr3J/xJATVGMgInHMmzGJRbOnROyLEPKXXeVRO0FGG+88yMzY62cYMqHZgIwPwYxsPKFhSSOJNZdH0MZDNazbXIpzLqGOv0/tLKeisZ3uHseOKIFBvM+wIWxW5s5u1xtMRPLqgWrKG9owG/i54tVoPL079gzQMHAm9dEipYGBc64LuBl4EtgNPOic25nKa4oElfujBcxRYCAixC6VS2aTw2gZgeDlH9gYuaOkSCK2DWIm7UQysiHFlZGb2wA8uLEkbia3s7uHX798kHWbS2ho66KmubM3CB5cMOzt+6etic3tcayurXcmcvAy+zXNkZs6dfc42gLzkIRLxrDBT+5M3ZwOqZTyeQycc+udcyc651Y4576b6uuJBJU3tDFpQhbTJ+WkOykikgG+9sFTBqxbmj+Fc5fN7l2O1m+gtrl/ZsHhKKoY+kzGKRhFUsaRHUfjT5zVEWGW4lRyOHqc6+17EPpTut8PghMNhiO1vXcRtoU7UNn/7zHa3/J9ftMhSKw0P9F4JjivSXA0qGgFEuHpzQSa+VjGtOMNbZwwfdKgOgiKyNjkHFywIr/fQ3p54VTOXTaby0+bN3D/sOUndh4fkCWJNFdBLI9tS12nSxldnHMpb0728OaRmzMjfEK/oNBcCj3OK7Dbc9wLajYd8kr4n44zutFwHuFmUOI38xvYeTjWnAiJXTt4jqd29X2O8L4Hj24ZWPMR3mwrE5ocKTCQMck5xwv7Kjle36ZmRCLSz7XnLO59f/7yfM5fns+MyRN618XKCGwrTWw21WidGEMjw/xlV/KGeZTRaV95U+8IOpkmUgY6Vqa1rbObJ3cej5nRDnlmd0Xv5GP7yr0S81gziEcSLPGPJDytL+yvSqj/dCLpP1rXSnXTwPQG5yqJdJ54Q5dmSvGlAgMZkx7deozrf/UGmw7XquOxiAB9Gf6srP6P4A+87YR+y4V5udz8npURS3P3ljcOa7bS0JWjZYRUuTl+jPSst4O53uPby2jp6OKFfYkNI1/b0hG10/BghKdwXViNR+hPssfFH341lDmPFtDsOtYw4DsJNe2J9nf4yoFqyupaqWmJPGHaYEr8HS6hQGSkKTCQMedgVTO3P1vUu7xg1uQ0pkZERoNgRuBjaxaxePaUqPtGG5UosevEPiYTmhLI6Fbf2smuY/37H3R0DW627dDQu4mMtBWudYgze0f602jr7DtX+OZEhl+N1VQr2Hk7tNfrB6tjhk8Hq/p3zO7t92B9a2L9hYcCkZLaFupaOjGjt/YkUygwkDHnhl+9wf5Ah8CzF89KY2pEZDQI5h8KpuWyOD96YDCY84UmVwpRjcD4VlLTwpHqxDPbh6ujjxAUzePbynjzSN9cGQ7Hw28eHbDfS/urqGv1St17wvrKlMQICCJlnIN/P39862hCfSce2zawzX3wsJinSGIAHelvMhkBerRzOBylta3U+bUOof4WmUKBgYwpzrkBJRxrliowEJHoImUMzl+eT0728B+RJTWt/ZaTMQyijF51LZ3Utyb+O/ByUeyZc+OJVUPV1N7ZO1zn3/+/N/tte7GoKtoJI64OlYQ7XL9rOucFN2X1rQOOaWiN3uY+eJlNh2r61c4NJrgO//yRMuvh60JHJDOID6Y/vEN4phUWKDCQMeOuFw6w7Kvre5enT8rhs+9cxswpE9OYKhEZK8JLQbcOYhx5kciiF02HSpQ7u3sGlOgnIrzZSywltYnUYsROQ3gGN5QZrm7uiBkExLO/omlI/TEGzkjc9/7+jUf67RdMeuiofrUXYf/HErpOpAy/Yf2GkA2fiTkTWhIqMJAx464Xinvf//oz57Dlm+/nm1esTmOKRGQ0GGqzgZ3HMqsJgIwu8fqorN/uTZD1yoFqjoWVuPf0OBrbOmMOR/pytFJ/vI67LxVVxfzd7+xyvZN0hWdyj9ZFDiSS1Zl206Ha+DvFEK8UPpQ3D09vpPSH1oW2DCY4GNStJUOqDhQYyJhRmNc3+tC7VhYMGHlERGQ4hjIfysGqyBMYdfX0sD0Jo7jI+NTa2c2zeypo74o/gdnWkoG/Z8fqWnsnDov2a+1wNLT1lfQHg4gX9vUPOpIREATPsa88+pwIQZ3dPRH7YTgHWWZx+zpEq4kYzNwFUc8ddfbzzM6bKDCQMaOysQ2A2y4/hQlJaBssImPXxJy+e0S8TMDb5k8HYo9wEs0rB6opjjC7aVe3o6giscyPSDz1MfquhEYoauvs6R0m9+ld5TgHzTHG1g9vivTgpuizFkfLYB+tHdi3YDjCr/PQplJuvvetiPtatPehpj5RMuhDqUGM1k8h3jHh957aKMOgjiTlnmRMaGzrpKqpg69cejKfu3B5upMjIhnozEUzyfKfxB85a8Ggjw81q0hEsJ3yL186OOhriQTFy6w+vj3+0J11LR3sONa/9uBPWweODBTyanH/js89YY3vwyf5ipTR3rCvovd9SU0Lzv8HifZrCF4gsZL2vo7QA/sGOGJ/lyMxt0Ssj7E+gZ9jqikwkDHhUJV3g1lWMLwhBkXGOzO71Mz2mlmRmd0aYXuumT3gb3/dzJYGtn3VX7/XzD7grzvJzLYEXg1m9gV/27fM7Ghg2wdT+dlWzckj229iOJhmQSvnTAOgszsTugbKWJJI6XSk39REZ+AO99qBvsz+YH6b++bu6PPkzoGzdwcz5SHmr39xfxV7jzeytcTrtP/S/qqIx8ZNRJx9ozfhiS9Uih8zeBjibWC0zFGiwEDGhGK/He/SgqlpTonI6GVm2cAdwGXAauBaMwvvwX8jUOucWwn8CPi+f+xq4BrgbcClwM/MLNs5t9c5d6Zz7kzgbKAF+EPgfD8KbXfOrScDff8jpw/52Fjxx2jJKEjm2XG0fkh9VcObB5kNHBI12mg+yfh1Df+dP1LTQoXfDDiWobTLdy5yMND3eVy/c4cKC+J1MnYEmyMNvGZECSY/E+4JCgxkTAjVGCzNV2AgMgznAkXOuWLnXAdwP3Bl2D5XAr/1368D1pr3RL0SuN851+6cOwgU+ecLWgsccM4dTtknGILJE7L5wNtOSHj/0MN78+HoI6fE64+QIQOQSBoN93cgWZnIwxEmM+ubyDf6RUpqW3rnDYo0G3gin6+8oS3pc3sErxue2Y/ERah/iN/5OP7+ifx8gvtkyi1BgYGMCYeqm5k/YxKTJmSnOykih4+MuwAAIABJREFUo9kCINjDsNRfF3Ef51wXUA/kJ3jsNcB9YetuNrNtZvYrM4s4G6GZ3WRmm8xsU2VlZaRdhsXMmDwx8XvHnc8XAVDTHL2j4FDzbAoYxr7htmPfXdbgTyQ2lGvDNn/+jbgj9sRJZvhkbfHG7/euH/2k4bUC8b6lrp6eIXfW7Z3ELEJ23DEwwIn3XQS3P7en0jsm+AkiTayWeHJHlAIDGRMOVTezRLUFIsMVKasxoDAtyj4xjzWzicCHgYcC2+8EVgBnAmXAf0ZKlHPuLufcGufcmsLCwuipHyHTJ01IynmGkjGINTa9jC6JlviHZ7Q3H671m8kEZhiO1ea+t8mMZ8fRBszMbxITfdz+oQp9rv4Td0Uql48enCQS9DS2dfFaceSZocNH/Ik0aZlhSe9sbAbH61sj9smI1vwouC0TKDCQMaGioZ0TZkyKv6OIxFIKLAosLwTChy3p3cfMcoAZQE0Cx14GvOmc6+2x6Jwrd851O+d6gLsZ2PRo9IqQOQqKlg+Il1k8XD3IkVwkowwlA1jR0B5xfaxzJSOjGX1+gwSODT9mBIvH+wVMgQtHG3CgtzZjiIkcEFyEgq6MaRw0OAoMZNRzzlHZ1E5hXm66kyIy2m0EVpnZMr+E/xrg0bB9HgVu8N9/FHjWeU/UR4Fr/FGLlgGrgDcCx11LWDMiM5sXWPwrYEfSPonIGPHMnoGj/4QbqUxovGAh4qhEgXkDDBv6qD5DO2xAmuNdP2Z/hASbXw04R1hzpNB5MqmmIESBgYx6je1ddHT1UDhNgYHIcPh9Bm4GngR2Aw8653aa2bfN7MP+bvcA+WZWBHwRuNU/difwILALeAL4B+dcN4CZTQEuAX4fdskfmNl2M9sGvAf4p5R+wDQYzIM/NBEVwNG6VnZoZuQxLdbvxrN7ynkiMG9GKFP558A496kYUjOod7SemCXqwRL5/v8ndI1Edx5iBjpa52NvjoPA8KeO3uZV+NvCP2/vaEYRvofwjxHqYxDaM2LzoYQ+wcjLSXcCRIYrNJNjQd7ENKdEZPTzhwxdH7bum4H3bcDVUY79LvDdCOtb8Dooh6+/brjpTQfnoDTO5EyJ5MvCMxhb/PHdAdo6u2mKMSutjB3lDW109zimTcphQlYWkydmc7y+nVq/c7th7CtvZHH+FGr8zrbBITOP1/cN93kofDjSsGv1+5WL16E2kLHdeWxgkJpIp9zQNc2go6s7btAQbajUoXLO+YFNjE7PCVwjUsa//3USTI/fyCjqUKguWnetkaMaAxn1qvzAoHCa+hiIyMi4+d63hnX8YCZYCwkVgsjYUtnYTmVjO7uONXCsvjXiPhsP1QxYF/oVenZP3+zCrxyI3Bm33wG+8Ey4w/HmkVr/ff/mSeETqg04NspwpcEM87o3SwcmKSx5/Zvb9N93sLUgffMRxJ4YLXzI0Ej9A/p3pE70+lH+xi00QlP/jtqZ0qxIgYGMepVNqjEQkZEVLN1PlfCM0F92xW9rLuPDgEwzjpKw+Qjils4HMqNm0NrRzf++2jfFSNT+BBGH3nSxtw86Ux9qvhRaHhoX+L9fEBK8RpyhXwcMPUr0wL5/hUyUD+1cxNGKMkXKAoORnupexq+yOq8aVX0MRCRTJDLCSVeP4/Uowy1mWF5BUiTR/HKk/cJLpF/cX8Xh6r6mRJGGDY2ls9tRVh97FuJYv5fRSsiDnY8jifa3cqCyiZ6wbYNrshN7/7iTmCWYYw8fBSk8AIkmE2Y5jiTVNQYZP9W9jG53PFfEd9fvZnnBVPIVGIhIhok6XKlBd4+jpLaV/eWNUY/v7snQ3IMMWlVTe9Rx92OKNKtuWPOYUN+CxflThpq8AZeMmakOK9GPlC4YXmn4m4dr6eoe2Lk5dP2IE6lZlPdh+0Rt40+c4CfKwYkM7dr/Zxb7uHRSUyIZ1X745F4Azlk6O80pEZHx4tGt4VM7DM/GQ7W978MzCg9sLEHGhq5u169DuUFvCX0i4V9DW2fUba8V1/jn9H6Bnt5dTltnt7eu3+/U4APNwWZe43cwHrzwYwY7MVm/yc4SrXUIzoEQNjJTvHkawmtHLDwy6d0v86Q6MEj7VPcydlU0eDfUudNz+cIlq9KcGhEZL1r9DFcsQy0RTHQ2Wxm9gpnIDXsqEs60Pra1rN9ypCY63T09tHZ205zAiFbJmgU45qzL0ZrZJ3CtaHuYxZ4LweF6vxOz8KY+8dMXPpRpoukdcJ5INQsRQoGhdGxOpWEFBmb2tJntiPC6klE61b2MHlv9URLu+MRZzJsxOc2pERHpM5RSSejLgAxl1CIZeXUtHdT4w4pGExp1KFx34GdfVNGU8MRasX41Khrbe/utRO5jEFaSnUCZdeymNfDw5tLeHaP2I4h2rtgD98S8bqK8OQkGXqtvXoL+m7zlwf39hTdhijWcaaTRkoLr021Y8xg4596XyH5mdjfw2HCuJRJun98u95R509OcEhEZy1KZR69r6WRqbvxH8fE4nUIlPUprW+nuccyeGn1UvNDPrjCvfz+4BzaW9GYNNx6sYdGsBRGPj/TrF69tf++xMdvTR94ysLQ8/Nr913R2R782xMlkD6KIPDzzHi/4jt1HInxUphhNf+KcK9a1o33yvuAg86RyVCJNdS8pVVzZzAnTJyX0UBURGarcnOxBH5No04Pg7LYQLOV1/XINwbHqn9+nZrejUTICzEQyxJBY06BExs638P/D50KIk5ZkB9WDyaDH2nVgxr+v78CAycz6Ne9LXLxOzhGPGcT5UyWVfQzG/FT3kl7FVU0sK5ia7mSIiAww3KEIXyuuYd3mgRNCARytjTwJlmSe8LkFogll4isa4wwXGq3pzYB2MhHfQm9TNX8p4eE/I11z4A5Ray5inCdhg4gyYtV6JDpj88DL9817EL4ufDnSNXprEWLOmfD/t3fe8XVcZd7/HXVLsiVZlptsuXfHPY7tOL04pJJGEhYIkGxgCbzLBpYlAZaaXVh2yS4LJISFJWQT0ovTEydOj4vce5dtWb13Xd17z/vHlHtm5pyZuUW6V9LzzcfRnTOnzcwtz3POU5KvGvSbYsA5/zzn/CzO+SLO+bWc82rvVgThnxMNnZheQooBQRCpR2WMwrshFxyv70AfhSodFIhP6WidNfTsB0caXFfuxURbALBhf2RnSNbOmllYLUTKzjgi6YhzUIRE9QrfKevfLXuxc4woiFKaV4VclToAG/X9TsNUrOx5FpwhSYHUDEuqgsKVEoOSo3XtaOnqw9zxI5M9FYIgiKgx5Ie9Z1ql5dprfzbgRPKwC3xbTjTLK+oYj/T5HfLdIPkYskg2HmZC8Gd2FPEDiC8ykR/8JB2z1Jc47GrlXOrgG6kbWbW3HMO54m8ZI4pdFC/nZE8fBcUY9oRuyYAUA2JQsn5nFdIYsG7h+GRPhSCIQcLEwpxkT8ECY0BvMOxLGCuvaBqAGRH9hRY2M0JXQAt5Kz75F3aeAWPwFWrULxzRrcqLEXmYIP76lVdVoUHdmnuaKcnaqMKgciGyl54Ezc0xW9VvvPK5dKdCMp41vYF7GNaBghQDYlBxqKYdB2va8NaBOpw9dTTGjkytH3qCIFKXC+eMTfYUYuZIXYf5OtpQikRyKa9w30UANEGyW1cWXtypJdBzi7/vaB/nSr+stdmn3eEYztV/e/uIs7JY5nzn9uc72dWEi0X+mmZEPnwAvMcc/FA4F2LQsO1kM2588GPz+B/XzUnibAiCIPoPt4gmBnvPtGJhacGAzIdQ42XWc6imHSunjVa29SOIcvOvf3Mc0UwoMp53O7Gt0UjWlyN6j0fIT1kde592nBY4EefpaHYx0hiXKgoR8yLrfNRmSv5RzS/V1XraMSAGDet3nrEcr5lRnKSZEARB+CNa0wB/QqLW6c7TLTHMiEgksSTCCnOOI7UdZlvRTEzuLOvE7hTsK1GZJESnRf63R/KxORH7eS+79SGflMspx/ZC9CK1q8mRR54D1bCyHBL2MjczJWvb1IMUA2LQ8M6hOowVEsQsnlSYxNkQBEHEhlucdJmAF9KjE5HT8eDDeGZv7a/VjjkQ5sCOU5p5EWPMNBPzE6rSGpXIOoa1npfgy6wJvlwHlZ/14yhvqS+G+fShyIgZiZntgmJJbiaPRmSt6PUIzEhObgqNzflZfD0YPsOkGBCDgpauAE43deP2NVMBANcvLUVaWqpvyBEEQThxE2oMwSFNkCqe3HraYatNpBb17b3u5zu08w7TmDi8Td3eBRbb+Vj79yEkG869fgV1t+vVnG8VjsIxvOdlArpsXnZTq2gdtt3n4D42YJ1fKqgNpBgQg4KDNVps6IWlBdj+g8vwy5sWJXlGBEEQ3kRjGmQtk9PcFcB7h+sUZ4n+4JNjja7nOSI7AnaMyDiWMmnOAP8Y9vXm6rVsJVwUOn0qH9E4O5u2+baQova5iGFFYzG78jrnuivC5PXsArmhQCUi14C9i6iea/zDJwRSDIhBwSFdMZg7fiRG52UhI53eugRBpD6xChuqldNQmCMQDGPbSc0UxWulmoifEw2dUdXvC4VR2axlPBZX0+1+AX4RHW1VpihS5VJi9+ImqLo53FpWtXmkXPtri1oUhclPrMgULhX+HLX9dqYw3RKUjGhIFWVAhKQrYlBwuLYdBSMyLT4GBEEQgxGZj4GR6Cxi/iAPS2QIOcZiiWqlGgBe21Md/2QJJS/tqpIKp72C4vbuoXrUtfcAAJ7aetozGy4HR18o7Gt8M8dAggxQxF0I4xiMJTSDryNykYdozJWvY7tmmcJtz12gmpF5v7lkZyBB+Q9SAVIMiEFBRWMnpo3J86/VEwRBpACu/gTCKvDuylbzdaJo7upLWF+Ek/Ye70RkhuM4g+Z0bMDtfwVh86ny045+ZD4mXr+HXgm1LO80Ycnb766EzDbezcE42vd2NCFZvfqwZ0rmUO06+DS7UozLzXPRJStLJdGGFANiUFDR0IWpxbnJngZBEERUSEMU2sMd6sflFU2oaumOnEghYYGQI4a9rBBMjt7Ud3Jkj1D1WB2OyYq4+xzcIYDLbPstk5TOnQtZjt1JhJAu7VcR3tRLqfHVN7cqXl5jR8plOwKxKewqRSyVoxNRgjMi5ekNhlDV2o0pxZOSPRWCIIiEc6KhEzWtPeDgCIZTV2Ag1HBwPFV+GpOKcsEANMTo+yEzrbH7D/vJKeBlomNGLfKyjefcdWdCbm9vDakaq/Ox2zh+Qo9q9SSmewxgStMfFpdZkMr52Bpm1nDc5uY4tGNAEFFwqrELnANTx9COAUEQgx+7ENAVCKGlO2CtA8h9DOAuqBEDi/Eowhz4yBa9qDcYAmCLfc85XtihJev0smkXMTMeW3YK4Cizz8t+4FfWdQ3xaY4vCLuquo7dMXkGYtW4YmboaB2nvaIaxaSoSHcy1I7Q9p2GyA6NvEE84WsTBSkGREoTCIbxX28fQWY6w9LJRcmeDkEQRMIwZIC8rHSMzssG58D+qjatkDmFQVFm6AuFpWE0X9pV1a9zJjQqPCIVcQDPbosoAKLg2BUIma/NyD5RyKjmCrzFZl5ukmNp5+noK2vEzFV/mclSpK2/MdzaJhrjvlt2W6DeCXBXJDzunbRP+UAMPpS5JEKKAZGyPPDWYcz+/mt4eXc17r5oJqaOyUv2lAiCGKZcOm9sTO2kkWfsq6jC6w+ONMj7Mf9qSaCCYY7ath5HPT8OsUT8fCwoZW8fsEaGkmW8lRWoohJ5YY8eJK2jh0nlLpKwWKzKF2q1kefS17K6snH6w67eT7JAA79yt9/sx1pduemXHyFf2jYFtANSDIiUoq69Rwvpxjke3XQSADCjJA93nT89yTMjiOEBY+wKxtghxthRxth3JeezGWNP6uc3M8amCufu1csPMcbWCeUVjLE9jLGdjLFyoXw0Y+wtxtgR/W/KbgtmpPXfz6VdFAgEw0rJzzAlspscbNhfizD5JySFmrYeqYT60q4qhdOrHIcOIUQfipgdCWZBkrJYYS79qIRveWI+xSp5gnIOWBOqeXeqnLtlF4Hrjthu/ZjajXRs1QKAStDXlDanc0Hy1QJSDIgU4/HNp/CdZ3dj+6lmNHUGcN+Vc7HhnguQm0V+8gTR3zDG0gH8FsCnAMwHcBtjbL6t2h0AmjnnMwE8AOAXetv5AG4FsADAFQB+p/dncBHnfAnnfIVQ9l0Ab3POZwF4Wz9OSRK52umWCXdkTgae2VapjakSQvS/aULDmrYePFl+GoFgCJxzUhIGCKXAy4GO3sjujV1AjfgHyM97Itjea/3Z3yPR2bCLjsj2cj/zszpHaweyzMNuK+JWtwj5SnxkXi5+CjIHbtsujnoaXK7Mxfhxkt3vFNgUcIWkLSKlOFLXAQB4bPMpAMCUYspdQBADyEoARznnxwGAMfYEgOsA7BfqXAfgR/rrZwD8hmkf0usAPME57wVwgjF2VO/vE5fxrgNwof76EQDvAvinRFxIoulPOdu6ChoFDkdN4NntZ5CdmY6OniDysuknvj+xO4wbVDZHQs46fr5sgqLbqrI3enQbn868DsdexbT8rIbbUWVejiyKc18r/H6cnpXnXXwHuJAzjkEPS+RpMmQoONaKsfhvKJWuFHA2tkM7BkTSCYbCONHQiXCY42itphg8t11z2ppaTH4FBDGAlAIQsytV6mXSOpzzIIBWAMUebTmANxlj2xhjdwl1xnHOq/W+qgHEZsg/ACTy99tmlWBNcuVhg27Ud5tPKMzxws4z8U6T8CAUEh6k8LDsfiKq1Xi4lKuIOKS749WvFiDJqaSo7PItjs6KN58ZhhPedaPFKzypQ/+ytVHtfPh5LtbISNxxXiQsDKDtTMgrun22kw0tJxBJpTcYwuf+ZzO2VjTjGxfPxImGTksUgbLRFKKUIAYQ2c+V/bdKVcet7bmc8yrG2FgAbzHGDnLO3/c9KU2ZuAsAysrK/DZLKP0dRlAUwGRRiCLz0P8iYhOdCiEOhyPF+dnKc8pVfCGkkCac29o5zc6dvge2qERG0jPVHNzeHYbgajf5sY+lnIzjdMQvQjWfREi/fv0qtBCxTodgzSk7ivG0sExGa/3/amds2bMViWXXYaCIa8eAMXYzY2wfYyzMGFthOyd1QiMIg3CY4+aHPsHWimYAwH+/cxSBUBhXLBgPAJhQkIMRWeluXRAEkVgqAUwWjicBsMe/NOswxjIAFABocmvLOTf+1gF4HpqJEQDUMsYm6H1NAFAnmxTn/GHO+QrO+YqSkpKYLy4ewjEK337iq+uyCwBNYHi6vNLTFlsUIJ/celpZl0g8oiJmCnOWMrGuoPR5dawQzvsbV/MdYf5eYT5N3wLLOQ8TG9tdsTsF9wd+krI52jjyEbgTUdqcY6eKAqAiXlOivQBuAGBZ+fHhhEYQOFrfgd2Vrbj7ohn44DsXmeU/+/RCrJhShAc/tzyJsyOIYclWALMYY9MYY1nQvsfX2+qsB3C7/vomAO9w7VdzPYBb9ahF0wDMArCFMZbHGBsJAIyxPACXQ/vtsPd1O4AX++m64qYoLyumduqoKOJqo7U8FFabKxgZa0UM/4dDNW0xzZGIDkMR++BIvaVcJu55+RiYxT6FYL8Zf1XzUtWMxrlYnIc9S7BqLn5zBxjVdLcEx46GMAH5qjvzdy+Tucmm8sfQ/iZ/9y8uUyLO+QFAqhHG4oRGDDO2VjQBAD6zYjImj87F+q+fizDXtmef+bs1SZ4dQQw/OOdBxtjXAbwBIB3Anzjn+xhjPwFQzjlfD+CPAB7Vv9eboCkP0Os9Bc1ROQjgbs55iDE2DsDz+u9EBoDHOeev60P+HMBTjLE7AJwCcPOAXWyUlBaOiKmdnwgnllVlxrDnTCvOnTkmamFr+6mWmOZI+KcrEDQVMcPJWGaVItrrS59eNHFMfTR3dRiOI4BHxA9GssruGFOitHDDylC9Uh6JWBTDxFwQzZJd+1Z8RqXRjRTXYDehkpmESWV+xsCYPBJSsugvH4NSAJuEY5kDGzHM2XqiCSUjs00/gkWTCpM8I4IgOOevAnjVVvbPwuseKAR4zvn9AO63lR0HsFhRvxHAJXFOeUC4atEE/Pc7R/ulb3tyqYWlo1xDRwKpH/JwqPLCDu/M0k6B0EUodmlrCuOSMtHHxF3gVqy4S8a1OvM6c2VEi5sZXSwmNXYzI7coS+JYon+BRcExy+Tt/e7iyP07DOXQOhdLnRRSBkQ8TYkYYxsYY3sl/65zayYpk94CxthdjLFyxlh5fX29rAoxRNla0YyzpxZROFKCIFKeueNHJayvfVVykx8z4gmYU2ig78mUxC26TFQJw7j3Obexnc6vdk9b+U5VpL5gHiQ4v7spH9Fm943HZ8BzLHEc1f0ylCTpToC6rZuMYihPkXumKSy+P62SyFDJxnPHgHN+aQz9+nFgM/p/GMDDALBixYoUuCVEf/Pw+8fw+/eOo7EzgDvWTkv2dAiCIJKKaidAFbkkFeyQCSvqR6J2anUIkD6lSYeAH4XALYt+pR0zqaLhFSHJNBTiglLreza2XQCPhtHoxmIOBbf7pTLrczMDiubjJ/U7UT3oFNH9+yuPgdQJrZ/GIgYZ//LqQTR2aolh1s4ak+TZEARB9B9Rx6lXKAPSuj76q2vroSzIceImeGsCqFWItsXZUfYnmpp4PSFZzoFYsDq6u8uihlAt2s1bztvre/Qlm4MXypwJfsOVuo0nWGr5XeXnxtNT5iewmm4Nxk9evOFKr2eMVQJYDeAVxtgbgOaEBsBwQnsduhNavJMlBj+t3X3mB+a3n12G2eNGJndCBEEQ/Yh7KEiJQ6dPRcJNMBL72HCgDk+WUzjTRCIqCm/tr5HXsYe3dEjR/p6fViC+TOyysj3UqHX3Sr36z4W7wG3llno2Mya5z7VkBd9yzbKJR3cv7Cv+XhmWVeZDXFAm5OeFa/E5n1QjLsWAc/4853wS5zybcz6Oc75OOHc/53wG53wO5/y1+KdKDAXKK5rAOfDXv12FqxZNSPZ0CIIg+hU/AoAoqKhMPXyPB2eEE86BxzafjLovwps9Z1o9ashX261nPeBWQVVMbBbpx75P4baL4TKUIl6/6r0YjZriJchH46Dr+3rc+hNW9Z2fmXj8IVQ7fCmsDQj0lykRQUjZdLwRWelpWFpGEYgIghj61Lb1KM+dauoyXzsirijaWPIYcGu5s16kwhNbtF2D3ZUU0jReRAF3QoEWxta5Q+AvGpDfcexOrmLfflbAPcdiTod3w5HWPl40Y4j3IR7BOBY53TSFEo4t513a2Mf0eo6e/hGq/BMpGFSAFANiQNl0vAlLygqRk0n57giCGPo0dASU57oCVgtbZTI0ZVgaeVs3WWPvmaGRBO1gCiZzE2+73YRGWp/5E9oBycq3/dhh/++1U8Fsx6p6krlYlFhnZb++BvEQMWNSewXHqkxIx+OxmXE5rcKszgwMzkWBZEOKATFgdAdC2F/dhrOnFiV7KgRBEEmlpq3Xs05Mgo1llTn69oOF7SdTZ+fDzQ7fj+hvNw1yhqqVRAOKMYJPrNF1vPpNFHYnZeV4tsV2mQuH6loZc78O2c6C1WnbGs1J2oebD5DYl6I8mZBiQAwY+6tbEQpzLJlMigFBEMObjQfrLMeGsMJtZRY4N1cYPRNWiQpCCqxCDiW87qfVeZX5el5emCY5+n/Wcx5tpf1Zj9XKjY+5WV7bfB18OB+7EXP0JduuRjTmVGIkJnVd92NtDtF/8hLtXB4LpBgQA8bO05qT1uJJBUmeCUEQRHJxRm+J2EPLYsFHIy64CVOv7qkGAHT0BqPocehytK7dVz0v3wwzmI4tyo92znAEltiZS+pJ+7fVEzP/egnPqvCajqRoiuhHnu89wT+BS64iYuvv/S62J11zq+NHafBSgOzjJTSXgus5hTN38vUCUgyIgeNwTTvG5Gdj7KicZE+FIAgiqRyu7ZCWayEiuS8hxU3yEHcfDL+CF3eeQWt3HwDghR1nopvwEGXLiWZf9UTfDL+r2HKHU/hbvpa1g835OAGry9LdBMGkyYxK5EcId5mPn7Vzr6uxRxTSPifOVqokgc5VfnFXLfa5STOVS9qYoU59KnTJghQDYsCoaOzEtDG5yZ4GQRBEv7OwdJTvuqLNcyKyGtsz2Bp9dvYOrXRCW040+a773uH6mMbYsL/Ws05E4BMEap9Lv271zJ0GZYI7Z4Qi67z8v5fUUbCEnQMzvKeweyEZwiqMx2+SY/Yr7JJ4CepiXwxAmn3ngEeiLslyOBh1okXp7O3wW4jOHGwgIcWAGDAqGjsxpTgv2dMgCILod2KN/iPKFaebuwHYzDpEPwRJKMmIMAlTSglzmYHH4OdonXzXRcZpITRsNNS190qFOJVQbjiqqrIhi381m/bIqrf/bL6xS5HRvguceQ1EXwexX3s9a/tY5iw6H9udiVX1pTDmNCmyrd4LVT375lAL9mL+iFg+camwi0CKATEgdAWCqG3rxdRi2jEgCGLos2Ci/x2DsBjlRCg/UO1fuXCLNhOWmTmk0AqlX949VOddKQb2VHolKbNS1dKNj481AvA2C3HkBrCfl7SRPRuHEC7Y7Fsj5jhfM1t7N9zm5ya0+r1+r1wCkTnbBG+PXZNo38+q+qqIRV6Zk1XhXx3O17AmILQ4bqfIZ5IUA2JAMFbPaMeAIIjhQFqMv/L2xFgyJ2WfHZktw4JmYPSXCiuT0VLV0oMD1W2oa1cnjYuFD45qZkZ17T2WXYj1u6qk9cPc3UxHM1ER7NddlDaxjRthQWDm3HAoVtf3Yz/vbQYU3fuEMflKuraLwn0JvlYfCvfxfe9CiI7bEidoDqupEbc9X5XPgtucvN2DIuZLqaIQGJBiQAwIv3v3KIpyM3HhnJJkT4UgCCKhnFVqjbR29aIJuHbxRN/tP9FXn2XCxDPbKpVmGm6IdUIKaeZQjb+IPKlEU2cA3YFQQk2jCkdkAQDae4Jo6NANO2Y+AAAgAElEQVTyS5xp6VaaHzkzBGt/NfMgo8y6ki2L1GO08UI0XbILzrK64rlohHtxVds49hvzX+YcrZ2P+F1ExhEVVWddX3P1PC/bSXAqa/YdCtFfRJb1OVqFWnZNqaYI2CHFgOh3mjsDeP9wPT57ThlG5mQmezoEQRAJ5eK5Yy3Ha2aMQX5Ohu/2BwUBPWKDrr3accoaJtNiuyyNhOI0wQgLq6Ti+a0VVufd+nbvpGvJpKU7gKbOgLl63h9+E5wDT249hfKKJjR1qrNWezmPGtGlZOcs49n/+pA8YxUs3e6X1HwGgkmMrY7bzpPUFEoiWNvLonK8dzFd8mteJA8f61bfbTrOhl6PKVU37UgxIPqVnr4QbvvDJoQ5cOm8ccmeDkEQRMKZWGgNwXzN4gm4ZcXkmPpSrbyK53z3pRC4DtfKdwre8hGBJ5mcae7G6eYuPLn1dL/0b5i8jMzJREaau3hkjxjksEE3Xxv11L4DooOtem428xebX4rDVt919s467pl67WPr5QpzKfs8Zc66AwoT1RtRqXGa2InEmyUa8O/bkUqQYkD0Kx8fa8DBmnacVVqAxZMKkz0dgiCIhDNtTL7leGROJtLs8RHjRBQaOyXJydyiq9iFmvKKZl+rzioFIlkYAma0IV1fUvgK2DEE3TQP8xmH3wciQrabQ3C0Jj1uY6rGj/RhtZF3Vzoi9eR9y+3tVf1IHaBj+DiY3TDnPZVJ3H7us0wJs2ZJ9vaFcNxryPMpiHW9HMpTCVIMiH7lvUP1GJGZjmf+bnXCfygJgiAGkgkF8uSMK6eNTtgYxrdkT19Yfp4xbDupJeVSCYuGCYghHIYlkpJR5OZnUF7hL/nXQCGLBiPS3tNnJnCzlkeR5Vnw6lUKmFxUALykfXnyq4g9v3qciPmXvNwv/hOyuXfi9K3wb/bkN2qRo53CLMh5T/zhFSXJCy/fEL/9qa5b9lkdaEgxIPqVdw/XY82MYmRnpCd7KgRBEHExOi9rwMb644cnzNchIaqQ4Rz73uF6qYBoCFInGjot7UXnVVHYWvef71vaH6/vwK7TVr+GVKeyWXMSPtnYhQrhusPhSHI3PwKbV5UPjsiTpNlX3I3M1QayHYZYEU2X3CJfRePIayCbV7S9WHNuMO0fnPdEVKyiifoTDc6+In42XuFG40F0yJY6iIv3yFaefLWAFAOiH6lo6MTJxi5cQJGICIIY4lw2Pz4fKm2FX36uplULz7mnshWPbz4FQIuaUyuE7bS3be7qM81AZM66KrmxL8TRF4rsVuyv8s6lUNfWg97gwGZVFoWrdw9FBHbxNrx3uB41bc7Qpm8fUPtSyB6Bcd9ON3VL5yEzcRHt6o1MyK7mSdzDBEYQpKNJVxevnbwm2OsCtdTUyOo+HY1DuGPVP0plxjXiks2WS2nq42O6Dv8N2Q6czRnb7fOsmmaqQIoB0W+8urcaAHD+LFIMCIIY/KyZUaw8d/5s7Xvub84pi6lvNwHBkJeauwLoDUaE9pONXSg/2aRoFWHDgVoz/j0Ay2u38QBgp757UN3qFIoNvvPsbrR1R2GuEwcRN1KnsKcJitqFVbd2AywSpUa83to29+hLbT195iq3HzjnsDoDRy/geo7hNhfOnUI27M7FRlWl3ZJrsXQ3wci5AbWZlyzzcTTCcDwr+UYOBW1Mp6OxlnlaHMvZ3q1vGdEoRqmoFACkGBD9RFcgiD+8fxznzRqDqWMoqRlBEIMfu5OxiOFCdf/1Z8XUN+eaAC835zCNVRzn7D4CRjJJo0+zD9M2mjmEFyMakdPkJXK88aDcjMYYJ9YQmo0d/kKkBoJWnwtpmExh9X7jwXqkMWbabNvn98ruasux4bcBAN95Zrer7b8+mBLRbMbTWVf2jCT29VYBVubJ6lSSVP26Ib1k5l7HayckVrSIRu4duF+T/n5gckVYfEay8Kt2p2T3kYTdIYUSFK3TfLIgxYDoFzYcqENzVx++duHMZE+FIAgiIcwep1YMblw2Cd+5Yk7MfauSkAHAbzYedZQ16zH293mY+liFmwiisCnLX9DT5980SGVLbfTTHVD39cY+fyFSn9lWCfEKni63hiyVOVGbK9mSHRK7k/KhmnbfycBUVZggEpoGNj6VJosgH8sqObcKttaQqdH3o83DYZGT8ORcqmt1i3AEqJ+BFafDtLwsdgVG7YRsKArccqy9jm/M/oYUA6JfeHlXFSYU5OCcBEbrIAiCSCYrpo7GTcsnSc/lZKa7LoR86dyp+IdLZ5vHq6dbzZKMFWwGoLJZbrZzpiVS3txlFWxlAtuHRxvw9LZKAMCeM62RuhJx5qVdVZbdhOe2n0mIQ+aR2g5l2NONB+t8R2ExHIwZA47UdSCkryaHdQcKccXfIE2yUuy2ahvL9VoSztmQrUKL5k4qvFanDdMo3xFwVP057o3idJQCrH23wlyRlyUUs3Vur+I2tOuztTsXMxfzH8fzcdaRZUaWYU0C56znTEAYvx9IoiHFgEg4nHNsO9mMtTPHUIhSgiCGFEsmW/Ox+A1Vetn8ccjMiHwfqnYfGANaupwhNwHgQLX77oDhNLz1hOZ38M7BOoTCWhnXbWPM/2wCiCykpyi07TilDl3K4MyiDADbdP8Hu6zT0NGLjt4gqlt78JSw8t8bDKFWcBb++GiD+XpSUa4553cO1pnlTwrtHSY3TAv/KF6HV3K0SOIypwAvnnPtQ2IeZMchACdIIBTNhrhtHKnAK1nJtpyX9G1H9F1REdnJcL851tPq/vw4LrsJ3HaH6niVLLdM0Hbc3hOJ3pGJhbgUA8bYzYyxfYyxMGNshVA+lTHWzRjbqf97KP6pEoOFyuZuNHYGsHgyJTQjCGJoYQ8Reaqxy7PNpKIRGDsyB+fOGGOWzRyrNkuS4SYvvHtIE5SNsKbiCr0srj/gFELcVqlDYY6DgqlOMOTMsbC/2rkrcKC6XSroHK5tNyMtiYJUW3fQdHaub+9FheTeMjDkZ2dE5u2QM+WmMMY83Ey2/OKwsRdfe0h2brbusr61884wlt4277Zjj3kxoY3avIdJX9uJdQXc3iOXOFUD7tcuVYRsk1D5FHgltZPPRY23jO9snQJ6Qdw7BnsB3ADgfcm5Y5zzJfq/r8Y5DpFCNHcG8F8bjqBVsaq1WV+tokzHBDH4YIxdwRg7xBg7yhj7ruR8NmPsSf38ZsbYVOHcvXr5IcbYOr1sMmNsI2PsgL6Q9PdC/R8xxs4Ii0hXDsQ1JpKzfewYnDdrDApzMzF5dC4Wlo4CABTnZ0vrNnQEpOXdioRnANATDIPziNArCvFiiE13IUZ0StbEk+5ACFUt3Xhy62lUNHSaOxZPlVfa5hbCU/pK/EZhNb+nL4STjV2epjOGiZSx4n+gug1v7KuRzzMKyYkJzsd+VrXFeoBMARAFY3U9VX/OCboeevbjd+fCj4mU2hxKG8hLwPdr/++poDhMcbzrSc+5mFp5ReVy9udd325qZDBYHI5F4lIMOOcHOOeHEjUZIvWpa+/BLQ9/ggc2HMafP64w7Tt7+kJY98D7uPXhT/Dz1w5iekke5k8cleTZEgQRDYyxdAC/BfApAPMB3MYYm2+rdgeAZs75TAAPAPiF3nY+gFsBLABwBYDf6f0FAXyLcz4PwCoAd9v6fEBYRHq1Hy+vX5gyOtezzj9dMRdj8rPBOUfZ6Fx8/6p5SGPA4smF+PpF/gI0BFzyBBgWm8frO5V1/JwHIoJNKBzG4dp2/O5dzfG5vTeIv3xyUtpmy4kmUwirbo2YAuVkpuPjYw2moFgumBuJQpuhTOiiFXackidYa+vpw39uOGIe7z7dKq1nkMYizsd+HYD9iHGWnQj7CTj7sDt9O9rZ6vhygOY2IdvlAqMJoekcx7lSboSAtY/hNxKSvb1shyHeDMKWPgTB3v432kRnKqVCdo8d7wPJfRPPpQoZ3lViZhpjbAeANgDf55x/0I9jEf1IbVsP7n/lAG5YVorfvXsMp5u6UTY6Fw9sOIyGjl7k52TgwyMNOFTbjoIRmeCc4yfXLkE6+RcQxGBjJYCjnPPjAMAYewLAdQD2C3WuA/Aj/fUzAH7DNAnlOgBPcM57AZxgjB0FsJJz/gmAagDgnLczxg4AKLX1OWiYUJhjOR5XkKOoCSwtK8SOUy0ozNUyJhu//ZfNH4cD1e146iur8LRt9V3FMReh3i0Drsim442W3YTtJ9UZjv9v0ykc0k2SDD8Ft2HCnONgjbaj8OLOM7huSamjzcGadmytaEZbTx/yszNQlGvNJO21mvzqHm0XwTAl2lfdajFZtQttaWks5hVbWStZ7gTznLk6b4UJWoQYKUjs0w/uZi6q8uiu3bS80YV8TZC19ud6/S4r9JCcNo5jlYn9Kg/M9tfeVpu6h6+Fx1iiomFVkOTXbu97UCkGjLENAMZLTn2Pc/6iolk1gDLOeSNjbDmAFxhjCzjnDs8pxthdAO4CgLKy2BLDEP1HbzCE2/+0BQdr2rF+VxUA4D9uXowJhTn47B8249FNkRWkxZML8exXV4MDyEwnv3aCGISUAhC9MysBnKOqwzkPMsZaARTr5ZtsbUvFhrrZ0VIAm4XirzPGvgCgHNrOgsPLNZV+Jy6aMzau9tcunoiROZkYlZOB7Ix01wWUkpHZ0lCidt45WIdL5nnPy/BBaOoIoLa9B89ulyslwVDYVAoAmEnVtp9sxpYTTfjF6wfxWVsit9q2Xmw72QwGhpauPmm406fKT2N3ZSuuXjQBJSOzLYqBsfvsV0Dae6YNC0tHWYRf+4o/AxB2WaV1w9Wsx+u8zYTHmlxLmKNlpdx6HZa5RLmqbZ+HZDgHKlt+z5FcTHbEftTj+rsWu9IoX7VXn3PzpzHuvbGiz7m+U6KYmnR3QDYfzpXKrqvpVpLxlN4455dyzhdK/qmUAnDOeznnjfrrbQCOAZitqPsw53wF53xFSQllyE01niqvxMGadvzjujn40rlT8fo3z8ONyydhzYwxKP/+pWa9+66ciyfvWoWM9DRSCghi8OLHv05Vx7UtYywfwLMAviksEj0IYAaAJdAWlP5DNqlU/p2Ixi68OC8Ll88fj9F5WVgzU3NEvvXsycr231k3Bz/79MIEzFLjSF0HAKAzEEKjwpcBMHIGRHj/iBYd6GBNOx75uMLi0CwmHjOEvH1VrVjxsw0AtN8QzrWd54oGbdfjREOnGUGpoqET208148ny0464+SKiMNbRK8+yzOB0JnVL2uZobzOFUUWyMZQYFdEoIn7NnOJF88tWC6qRpGr28ojjs6vDsarcwzTI+XxiQ/rlIygtnrtqUT4Dr7wTfaEwPhSiavkZNBUiEgH9FK6UMVai25aCMTYdwCwAx/tjLKJ/eXNfDaaX5OHui2bih9cswNzxEb+BMfnZOHemFov7zrXTkZOZnqxpEgSRGCoBiJLqJABVqjqMsQwABQCa3NoyxjKhKQWPcc6fMypwzms55yHOeRjAH6CZMg0q3H7Mx4/KwY+vXSDUZY4Qzowxx+q7wc0rJiPDp0nmE1vcw3CK7K5swWt75c69spXP9w9Hsh6/skfLt2DsCOw5EzFHMoS8xZML0RsMIcw5AsEwWroCeGt/Ldr0kKj7qtpM5cOoYwhx0diYh8PuwqrMafQZSV4HEXHl3o9SYRUQFWYoCqdlLywZdC3OvdyzLw7/JmbiWPGYtPgZLhbTrnjl5YFYhZclLTN2HtSkwv6Ak3jDlV7PGKsEsBrAK4yxN/RT5wPYzRjbBc0G9aucc2eQYyKl6Q6EsOl4Iy6dN05Z509fPBs7//kyyldAEEODrQBmMcamMcayoDkTr7fVWQ/gdv31TQDe4dqv33oAt+pRi6ZBWxDaovsf/BHAAc75r8SOGGMThMProUW6GzJcPHesL2Hp86umKM9NKvJ2bgbUgq4MWc4Bkae3efs9PLf9DADgxgc/MctqWnvwxr4aVDZ3Y96EUXhTz2r8yu5q/OL1g44+XtlThc5AyLIizcHxuhCRqLWrDwdr2qQKQHWrPBHcxkMRZ+Ywt9rFB0McHx1rwO5KtX+FF1KHWqnzqWcom4iQH+VysShwbjhQ51LTZ3+u5j52c6godkS8LsulK79O2NGMbTijR6ufRFvfzUQwEYkD+5O4nI85588DeF5S/iy01SFikNDa3YdntlXipmWTUJCbCQDYW9WKvhB3zV6cnZGO7AzaKSCIoYDuM/B1AG8ASAfwJ875PsbYTwCUc87XQxPyH9Wdi5ugKQ/Q6z0Fzak4COBuznmIMbYWwOcB7GGM7dSHuk+PQPRvjLEl0MSDCgBfGbCLjYP5E0Zhvx66c5Utg7EIB8eaGerzBvMmqCO4rZ01xlE2vSTPV4QhFX0huZTTHQjhtb3VvvrolURJ+u93tAhG7wk7DAb2BGo9fWG0dWtlhpD7/I5KjBmZjcaOXry8qxrXLJ6IB987hjJF5Kfmrj6njwHneP9wPWaNHWlmPubgaO8JIhzmCIY5XtldjdLCEdI+Xe3UJavq3OI04GxrCNVihBzpuKKPgYeJiXle/1Pb2oOcrHTL2DKH20Q4+XolQVONrerXa06ynRL7ObmiJh9P9lo1nlnXRx17v8Zujajwpeb+gJP+jEpEpDicc/zf5lN450AtPjjSgGCYY/2uKjz3d2uQnsawS080s4jyERDEsEEX2F+1lf2z8LoHwM2KtvcDuN9W9iEUMgLn/PPxzjcZhAXJYEaJOlHZ2FE5mDYmukRmkX7zlOfuXDsd9z2/J6Z+3eAcCEiSl8kwfh/8EFZIUka5HvTIzMcQCIax43Qzrlk8ER8ercdnV07xuUqt2cNPG5OHgzVtOKu0AGHO0dkbxO82HsW2k81mLoa+UBh1QpZlQBPiDN8L1a5AJGOuu62+Eu7MOu1wrPWxOv/IJydx47JJaOkO4Fh9BxaUFij7FG3t3ZBVsSoWzqhEMkXK6ZTEIqZi3tOItHNI5EKEI6aoYymLRASy30eHmZXPexQx43KvbA8akCr+A34gL9FhSk9fCHc8Uo4fvLAXe8604oZlpbht5WTsOt2CP39cgb5QGJ8ca0Rp4QiUjJQn4iEIghiOrPSR1AzQIhjFGrb5v25dqjx3i4vDcqx0BYKobO4yTYS82HG6xeJ74MZb+2sxQuKDtq9KE9IN0ybD+fhIXQd6+sJo6OjF8fpObDvpCFTlSkZamhYhSfdZaOgIoK0niLf215p1gmHuMMF5bU+1JbGaamXYZ/oAC17KgI8eAAC/23jMUvranhoEBWdoe/Sdxk61k7kffF+fIv6AmTdA1b+qge2cm2O6pGm/4XeMtw/Wup53vh+i678/oR2DYcob+2rwzsE63H3RDHz78jlahsgwR0VDF3768n789GUtxPg3LvaXfIcgCGK4MNmn3X805GdnWKLtLBRWgHf84DIs/elb+M1nl+Lp8kqkpzFMG5OHOeNGWuzx4+FnrxyIqv7Jxi786KV9vup2BtTJ2QCYwitjQGNHZKX1V28dBgA8u70Sy6bId67fO1SPC+aoI1VxyIXbps4Aatp6kJOZhsIRWegKBPHf7xy1RFwyZLRuff4jstLxxr4azBqr7QIdq+/AqumjwcHR1h1UmrRYVpcllSwOz1y9Gv3BkXqcaemOysZ/28lmV3Ngc1r2nQAXUyprmT/81ItnVV3eVm6eZd/58Hs33SIrqQR6mfOxW0bmVIB2DIYhvcEQHv3kJCYU5OBbl80xVy7S0hj++MUVuGbxRLPu354/PVnTJAiCSEmMFdpf3HhWwvq898q5eOKuVdJzI3O0NbyrF03E1y6cAQB44JYl+N5V83DlWbI0Q9HhJ1eCjHj8HGRz2Hm6BT9+SZ73TiU0vSnsAohRjVq6+lDf3osw58roPC/vrsJre2rQ3RfC45tPWZQCAHhZz93z3uF6/PHDEwA0hcKgS1B4jjd0KOepEuTtTrCRkKFOYxzOYSan+97zex1tjFFljsiHhKR2jrl5LFFrkXVcq8RFrDF77FGs7OfUfXLzL7NvSbiMaTrIK2T3oG4PZ89h4UYq7A7IoB2DYcgDbx1B+clm/NuNixzRhHKzMvDvNy9CRUMnbltZhlE5mUmaJUEQRGoybpRmXuk3YpAfFk8qxIKJcidkMWz+Obqz8xI94+9tK8vMjMAqzp9d4mr284cPkh9N/IENh13P+1klr2rpRnqaFtmorr0HZ1q6TaVKRl5WhhnNKWjLTfCnjyrM8KqfHGu0nN98oglTx0R8QBiYKRjK/ClkNv52gZzb6suwJ41jzLkSX9/eg8lFIyw5HbafasbMsfkuplHMMQe3nA7SfmIQct1E50hyuHj6ifRh8Q/Ro18pW3nI9PbnGQ5zHK7tsIzarT+rmH1RkgjtGAwz9lW14s8fn8D1S0vxGYWdanZGOl76xlplbG2CIIjhzKeXaAmdE7nit7C0wBQibl4+yXIuKyMNl82Xh43OSLP+jE8pdioruZnpeOxOewLrCKqEYamEHz+Df3pmNxo7Ani6vBJ9IY6tFU34/fvHfRlonGrqshxvOaH5Pfx241GH0nC8viMSeYdpQqaRVfqJLadNoZ/poqj9bWI/7nIxtRLfY9WtNmdpWAVP4x4Zuygh0fcgDjMVBua5kg64mMgww6Qqcah2C1R41o0i7OnJxk5L5KUPhERmbgqHr/CrKRC7iBSDYcCB6jb87V80R+Orfv0hstLT8I/r5iR7WgRBEIMSY6e1P37EV0wpwq0rnYs2f/jCCmn9eRNG4isXREw+C3OzzNdZehb6uy6Y7rpyruK5r62xZF5W+VFX/PyqqPuOltc8dkUAoErPo2BkVW7XHY5VWZ6N3QK3HBBnWiK5EgzhujMQwvpdkbx/nKtDwNqx38ID1W1Yv7PK6mgribhztK4Ddl7fV4OdQnSoA9VtaBCiKoVcJFEjUZ0d+3vabWXbM5mw6Agd50cl2ubW8K/2vriptMVClU1BC4a875nfkKepYF5EisEQp66tB3c/th1v7a/Fo5tOYmHpKDz91TWYqIjhTBAEQfgjkaZEBn+9axUWRxEiujA3C9+8ZLZ5/Ocvnm2+fuHucwEAy8qKMH5UDi6YXYIblpb67ntZWZEZTejGZZPwN+dMUdZ96HPLfPcbC9196lX17z2/1xRo39xXa0Y7euegZm//ZLn/rNBuBMNhhxLx3qF6MKbtumw63oS2nj68e0gz2/KTobii0bpTocquXC7ZMbHnhjByawCar8GD7x6zNzHZccoabjbuqDhRSNmOHRSfY8aSNdnLByESNSmypxHtMBnp1tCw1rG8dxBSQRkQIR+DIcyx+g5c95uPEApzPHHXKoTDHIsnFyIvmx47QRBEPJxVWoBpY9S5BmIlMz369boRWem4ZcVkPFl+GkV5Wbhz7TTUd/RivuCzMHZUDr59+RwsLB2F65aWYsnkQvzrqwfwxFZ3ofnG5ZPwrad34apF412djVfPcCZiA4BJRSNQ2SzPUJxIDIdcv3kYYsFQNESqWroxbUyeacZT396Llq4+lBZFFt/EyER+THLsvCTsTqiw+x/Ix9BGae5y7qCIycIsEXskztHmsagNiPUkbWU47e+9vAUi/UbupxDRydG3t7+HtnvgbWrV2xeyKAAGHb3BqJzwGRgCwTDCnIMjMyV9DWjHYAiz8WAdOnqD+L87V2LV9GKsmTmGlAKCIIgEcO5MuSCcLO67ah7O07Mkf//q+fiX67WISW/+w/lmnbMmaX4MF8wuQcGITN9+ZLesmIyL545DVoYmMhSMcAalGKn/tky3KUtfPnea+VrlJzFYUAmAYsjYRj0MqoGRjdk8NsuZVSgUotkwMOyubEF7TxCdvUHHzoCMgy6RhywDA3h9rzbfnZXWXQND4HZkGFaoLWJ5mPOIQmCvx4FQOHqFTaVY2Fw+0NgZwO5KpzmYXamxOAwLfiBcewHhj4NnFfk9vJTeD440WI45OE41dWF/VZujbqooCaQYDGH2V7Vh7MhsLJ/iLxkPQRAE4Y9Uy/FSMCITv/lsxJzHWASaPW6kso0qq/1tK8uwsDSy2/Bp3fwoPzsDB396BS6ZO9bRJi2N4RsXz8RPrltoKZ8mZHC+VgiFHSuXztPGXlpWiKsWTYi7v2iwOygDQLuH47bbarLbinpVaw+C4bBnDgg/NHf1WZSLEw3anALBMN7Wd0HUMfqdYVUt5zl3Zm6WVH5hZ1VUSd0cuxPC8VPlpy1KSU9fGL3BsKOur2zPljrRRyoSzbf8jmmMJM0LkQLKASkGQ5h9VW3K8HcEQRBE7KTi7qtsJT8WZpTk4dm/W2Mer56hhUi9Ydkk5GSmY2FpgTnW7HH5Zr2bl0/G2lnWnZSL5ozFrXoEvKsXTTCzRhs7DFdHKdxPLc7DV86fjqvOmoB/vDz1g2gcsTkNy6LzGMrFlhNNuokJxyfHGpCeQCnxQHUb6tq0fBViFKqP9Ig60uRs3Ih8JBwLFRmYJbOyxZRHv8q9Va1RCLtWwbxJkbXZT+4Fcx7mToB13rLdGnsf9pCkopNxa3efw+nYYGuFewQtMZSsOMdUgRSDIUpNaw8O17VjaVlRsqdCEARBpCjvfvtCy/H91y9ETmY6sjPSlW0umTcW//uls7Fq+mjcc1lEOC/TQ6Xed+VcnFVagKn6sSFMMsaQoYc2+sHV8wEAJSOzMSY/2/d8v3fVPNx75TysnTUG4wtyfLcTuWC2OlNyohCdlC0r5S5CbUVjF57UfT76QhxpaQy/fvtIwua04UCtQymxm+UAQGNHwNxVcBNaObhpKsOYfFdhT2Wre8Ixl5MfHm2Q54CAdy4CEbfdELOOan76X8NcyZju5hNNSvMtWQQprS/dfIkxab6LVIEUgyHKy7urwHn0qzEEQRDE8MFI1DVrbD4O/OQK5Gdn4JpF7iY/U4rzsKysCP/xmSWWWPkG+dmZuGPtNLz+Tc2/QQxzOl03LbphmWaeNHNsPj767kXKsdYtsPolGEL23IqhHzEAABmGSURBVPGjkJOpVl7c+H+XzIqpXTRkpDHUtMozSu+qbHEIokb4UM1WXzurytgcKxzAXpfQrAbNXQHU6dmwt51sxsfHGgH4CV3qDNtZ1dqDRzedtM7DtO/3zvFgbaf9PdnYheMNHTjV6DTtso/hKFfVV5w/09yND/VdlZzMNPN8MEond1FxUu2EpAqkGAxR3jlYh7njR2J6Sb53ZYIgCGLY8sLd5+Lzq6dgRFY6rl08EQW5/kySCkdkYkSWU4w4f/YYfHppqSm4zxqbj3e+dQEA4B8unY3Xv3keMvToS2dPHY3sjHRsue8Ss/1NeoK3eRNG+XK89cvc8Zq/RXFelkfN+OkKhNDQoQnXopD6zLZKnNQFWpWgXaGv1qvyRsRDo4tQaqxoh8MwzZhkOwoA0GTLDZHGGGLwL7bB8JzNyVcV8rWjJ+KUHTF1sk+WKY/MqEZcVSMyg54+7cKOContDMXJTkVDl7nbYufZ7ZUAIqF3ZTseqbCRQIrBEINzjsc3n8LHxxqxNsWiZhAEQRCpx5LJhWYG5WgcRPOyM3DxXGekIXt+h8mjc81FquL8bMwdr/m+PX7nOaZz9NhROfjFjWfhp59eiF/cuAgA8Nrfn4ciQYj/yvnTESulhSPwbd0nIVYTpFg5WNOOvVXaSr09OZYMw+dAJWAmmt5gCB8dbTSPw5wjLS3if2Bn0/FGy3VsO9kMBniax2w72ewQfO3vNkNo9vMuNMYzVu8/VMwXcArhRphS1TxkZ043RSIQ2bNQG4gRqexUNnebylMiFd5EQ4rBEOPRTSdx3/N7AAAXznFGjiAIgiAIO9cs7j+z00vmycOUrrEtXt1ydhk+vWQi0tOYmc3533QlAQBGxeFcffuaKbh0/jjTh2L5lIj/nWha9Isbz4p5DBVdgRC2nGiylKlEaHGFXpZNOTsjdrGtpatPWr73TCSyzp4zrQhxjnTGLCFARcHaiLT0THmlWcaYc3fB7+r3y7sjeRrsAvrGg3U4Wuu02e8MhPCaHna1Wb8uI7GcwX5dGXMkHZMoCZuF57PnTItZw57zoSvGKFGGyZ09ipHBIa9wswMIKQaDmN5gCE9sOYW7H9uOrz22DT98cS8efPcYVkwpwmt/f54jOgRBEARByBiZk5iIRvFizOPeT80DoO1KfGaFZlp05VlO5eVH18w38zeI3HflXDCm2YUDQG6WFgXpxmVaXw99bjnmjh+JOeNG4p7LtMzRn1tVhnULxuO8WWPwyJdX4qfXLUjotdUIq8xhzvGq7lcQDUY26ynFic+6vVEPXcq5ZhpkSRZn5lmImOyIoVp3nm5xCNzbJNma7XBoq+e1bT2ob3eutm+taEZ7bxBBhU2TPY+AsSLPAewSFBvDdOiZbZWWY3GDbJee16G2rRcMQDDEHT40hnlYtLx3uN5RVn6y2fSTkCmBySL14q0RnnQFgnh88yn84YPjqG1zvkl/ct1CzJtAYUoJgiCIwc/ta6ZiclGuNNP0silFuGrRRFz6q/ewaFIBZpTk45xpo3HJvHF4Y18t7lg7DY98XIHrlmgO1YbfQ8nIbPzo2gVYWqblcrjnstlYVlYEBoavXzQT50wvxsPva6u7F88dK816HC1vH6g1X+841WImjIuGeRNGYc2MYjxwyxKc8y9vxz0nETHMpj0/w58/rsCKqUUIcy41hzpc24GFEwsQDgPd+qp6h0eOB1Eor2vvNe32i/OzTGdsg6N1HZgjyclhVwz2V2th2t8Qks6pMjbbRfH69l4E9HwIHPKkcZ6J5KLA2HnJzdbekx8cacD0MXkJCzscK6QYDEK+9dQuvLa3BqunF+Pfb16M/OwMZGek40fr96GsONdMAEMQBEEQg50540Zi5lh5IA0jSdtTX1mNrkDQEqL7L19eiZzMdFw0ZyxGZDkjGK2aXmy+PmfaaJyjHxt/p4/Jxx++sALBUBiHatpxpkUTQpeVFWL7qRZHf17YBepAMDZv3WsXT8S4UZqfRMGITLR2y02EEklvMGzxQ1DR3BXABy62/gZ7z7ShMDdTGvO/oSPgiNxT3doDDmD7Ke9diObOgOkwLLLN1vbFnVVYLbwHAGD9Ls2s6VdvHUae5D3TH4i+C498chJXLBiPz62aMiBjyyDFYJBR1dKNN/fX4s610/B9PQ60wZNfWRWV4xhBEARBpDoZ6Wmewsqc8c7VZCMJnUwpsHOOTUAEgEvnR3wj5owfifcO1+PHL+3HzSsmx6QYxMpd50/Hw+8fN0O33qInjNv343VY8MM3BmweXjR3BdDW3YfsjDT0SgRzg5d3a7sBKp8HVTjPioZOS4ZjFXYFrLWrD3nZGaYvRVVLNyYUejugq8yXZNy4bJIZdSheKhoHxulcBfkYDCI6eoO46cGPAQCf0b8YREgpIAiCIIjEM70kH9cunoh9P16H21aW4bW/Pw93rp3mqPfq/zsPV0l8IeJhanEerjxrPNYtGA8g8lufl52BueNH4svnOueRDDYeqkdnIGQmPZNxulmde8CLt2M05zpui+4khmw909KNDw7L5+tHCTFIZM6oWpfIRgNBXIoBY+yXjLGDjLHdjLHnGWOFwrl7GWNHGWOHGGPr4p/q8KSurQdffXQbfv32EXzl0XJUtfbgoc8tN0O8EQRBEATR/xTnZ5u7EPMmjMKSskLTfv3+6xciKz0N8yeOwn98ZnFU/S6ZXOh6/raVk7F2Zglu0B2nRb6weiou0c2HN9yj5YowIjoBwL/eII+y9Nid50Q1x2gxzK7s2CMHJZtTTV0Of4pYuGiu1YR75dTRMffVrNhJGSji3TF4C8BCzvkiAIcB3AsAjLH5AG4FsADAFQB+xxgbGGOtIcafPqrA6/tq8Ku3DuOjo424bWUZLpsvD/1GEARBEMTAcPWiiVhWVoSd/3wZ/uacKXjn25pg7paReWlZIUbYzt/7qbmu4zDGlOFkbzl7shl6debYfNx1/nQzohMA3LLCaV0AABMLR7iOGS92xWD8qIHNHWFQ3aKtvu+xZXwWQ7TGynTdGX6Ffv9vXx3xCwiEwhYFbTARl2LAOX+Tc26oWpsAGOrsdQCe4Jz3cs5PADgKYGU8Y7nRFQiiK5C6ySJi4XBtO7799C489N4xXLFgPLb/4DI8esdK3P/phcmeGkEQBEEQAP73S2ejMFdLwiYmdrtCN/ux88xX1+C7giIwvSQPK6dpq8tijoKy0Vpfn1qo9aMKJ5uexpCTmY4T/3olAOC+KzWl4LmvrcG0MXlIS2N4UQ9xajCxIAdTi3Nx4l+vxK9vW+r/YmMgNysdk0ePwCYhs/VA4pZwLFYunz8Oj3x5JX55s5Zjwwij++PrIvLZC3efi9XTiwdlMJhE+hh8GcBr+utSAKeFc5V6Wb/w548rsPYXG/FU+WnvyoOAUJjj83/cjGe2VSI/OwP3XTkPo/OycN6sEqT1R450giAIgiCiZpRCYL9hmSbyfHHNVEt5ehrDzSsiJkFPf2U1GGMozM1EbzCMz55TBgBYt2AcfvWZxfjGxbPgB7uP4bKyIjxwyxIAwOLJhXjky5G12S+smQrGGBhjuGBWia/+RTbccwEe+twyAMAH37kIf6PPWcbPPr0Qd66NbeX825fPVp5LT2O4/3rrQunEfspobdy7R+/Q/t64fBIumF2C5VNG46I5Jbha2M0RE+SdO3MMrlo0AT+6xhooJtXxjErEGNsAQKb6fo9z/qJe53sAggAeM5pJ6kvduxljdwG4CwDKytRvLjdWTy/Gu4fq8Z1nduORjyswIjMd4wtysGbGGKycNhq7TrdgYWkBMtIZSgtHuG7z+aWnL4Qnt57GB0casL+qFZ2BEBZPLsQ500ajUneumVqch8bOAKpbe5CfnY62niBuXFaK9LQ0LJlUiIJc5xfKtpPNeGV3NWrbevEfNy/GhXNKUJyfHfd8CYIgCIIYGC5fMB4b7rkAM8fm41h9Bxo7AqaykJuVgcM/+xSaOgMo0ncb3vzm+Xh8yyksn1KES+eNxcVz4zcZFn0XLphdgsvmj8PZU4vwJcFZuSA3E19cMxUzx+bj+y/sxfwJoxzZeX/66YWobO7C7987DkAzWZo5Nh+PfHklJhaOwA+vWYDeYBifHGu0mBD947o5uGB2ibmjUpSb6Wk/f89ls/Grtw4D0Ey1lpYV4dU91UhPY/jLJycBADNK8nDvp+bh0vnjcLC6HWHO8djmU/j1bUtRfrIZP3/tYBx3Ddh07yW48cGPcaalG9PH5GH19GLs+MFlKMrLQvn3L0WhkGfgwc8tt+z0fGbFZFy7WHvOmelpuH7pJByt03IflIzMRn17L25bWYa/bjkV1xz7E8b95qxWdcDY7QC+CuASznmXXnYvAHDO/1U/fgPAjzjnn7j1tWLFCl5eXh7TPALBMB5+/xg2HW9CmHPsPdOKth6nedGU4lz88Jr5ZgKJ+vZePL7lNFZNH40blk7CeA+N82RjJ57fcQY7T7eYTjTnzizGiMwMHKxpQ2VzN4xFfSPSVUYak4a9Khudi5XTRqNkZDaWlxUhLzsDt/9pCwKhMCYU5GDjty9MiBJDEMTQgDG2jXO+ItnzSCbx/E4QRDLYcaoZs8eNNB2XVQRDYWSk91+wyKqWbmSmp6FkpHOx8XRTF94/Uo+rz5qIxT95E5OKRuDfb16MWx/ehIqfXwUAuOepnfjJdQuRL7mOfVWtuOsv23CmpRvjR+Wgpq0Hv75tKa5dPNFSb+p3X3G0ve/Kubh60USEwhzjC3Kw83QL/vxRBX5+41kWE6rt+n3knJvl7x2uxwWzrbse3YEQ7nt+D5ZNKcIPXtiL3Kx0dOlJ12aU5OFYvTwc6LvfvhDfeXY3Hr/zHFS19OD8X27Ety6bjW9c4m/Xxo17ntyJm5ZPwpqZWpbu3793DDcsm4SH3juGP354wlL3qkUT8NvPLotpnET8RsSlGDDGrgDwKwAXcM7rhfIFAB6H5lcwEcDbAGZxzkNu/SXyCz8YCuOl3VU4WteBVdOL0dQZQHcghF++ccgSqgoAstLTEAiFkZ2RhqsXTcTO081o6gxgfMEILJg4CmlMS429+UQTzrR0gzHNFvDKhRPw5bXTsLC0AICWJry7L4RM/YPdFwojTd/e23KiCVkZabj3uT0oLRyBKcW5qG7twZYTTegKBCHqDT+8Zj5uWDYp6dnvCIJILQZCMdC/1/8LQDqA/+Gc/9x2PhvAXwAsB9AI4BbOeYV+7l4AdwAIAfh/nPM33PpkjE0D8ASA0QC2A/g851wexFyHFAOC6F8O17bHFPkwHOaYft+rePMfzsflD7yPAz+5wpFDor2nD2EObDvZhI7eED44XI9f3uyM4tTSFTB3GmIhFOboDYZw/r+9i833XYJz/uVt3LZyMlZMHY227j5846878PlVU/DoJm0X4vefX451C8aDc26aZX376V34l+vPiilDtV8453hlTzV+885R/OELK1CQm4lwmMd87amgGBwFkA3txwEANnHOv6qf+x40v4MggG9yzl+T9xJhIL7wO3uD2HGqBWH9ujPT0zB7XD4efPcYPjrWiJauACYVjcD0Mfl4ZnslcjPTkZ+jacezxo3E8rIi3LZyMsYm0MO+OxDC9lPNCIU5xuRnY/7EUQnrmyCIoUN/KwZ69LjDAC6D5hu2FcBtnPP9Qp2vAVjEOf8qY+xWANdzzm/Ro9H9FZEFoQ0ADCNhaZ+MsacAPMc5f4Ix9hCAXZzzB93mSIoBQaQu2042YfmU0ThS245ZKRxWPRzmCIY5Dte2m4u7yURUSOIh6YpBokm1L/zath6MzssydwAIgiCSyQAoBquhmX2u048tZqF6mWkayhjLAFADoATAd8W6Rj29maNPAD8HUA9gPOc8aB9bRar9ThAEQaQKifiNIInXhXGjckgpIAhiOOEnopxZRw9X3Qqg2KWtqrwYQIsQ8rpfo9cRBEEQ3pDUSxAEQRj4iSinqpOocuekGLuLMVbOGCuvr0+tzKkEQRBDCVIMCIIgCINKAGKq1EkAqlR1dFOiAgBNLm1V5Q0ACvU+VGMBADjnD3POV3DOV5SURB93nSAIgvAHKQYEQRCEwVYAsxhj0xhjWQBuBbDeVmc9gNv11zcBeIdrzmrrAdzKGMvWow3NArBF1afeZqPeB/Q+X+zHayMIgiA88ExwRhAEQQwPdCfgrwN4A1po0T9xzvcxxn4CoJxzvh7AHwE8qkela4Im6EOv9xSA/dCi0d1thKiW9akP+U8AnmCM/QzADr1vgiAIIklQVCKCIIhBAiU4o98JgiAIFRSViCAIgiAIgiCIhECKAUEQBEEQBEEQpBgQBEEQBEEQBEGKAUEQBEEQBEEQIMWAIAiCIAiCIAikWFQixlg9gJMxNh8DLWHOcIbuAd0DgO4BMHTvwRTO+bDO8EW/E1Ex3K4XGH7XTNc79InmmuP+jUgpxSAeGGPlwz2MH90DugcA3QOA7gEhZ7i9L4bb9QLD75rpeoc+A33NZEpEEARBEARBEAQpBgRBEARBEARBDC3F4OFkTyAFoHtA9wCgewDQPSDkDLf3xXC7XmD4XTNd79BnQK95yPgYEARBEARBEAQRO0Npx4AgCIIgCIIgiBgZEooBY+wKxtghxthRxth3kz2f/oIx9ifGWB1jbK9QNpox9hZj7Ij+t0gvZ4yxX+v3ZDdjbFnyZp44GGOTGWMbGWMHGGP7GGN/r5cPm/vAGMthjG1hjO3S78GP9fJpjLHN+j14kjGWpZdn68dH9fNTkzn/RMEYS2eM7WCMvawfD6vrJ6JjMP1OJPJ7jjF2u17/CGPsdqF8OWNsj97m14wx5jbGAF133J9pxti9evkhxtg6oVz6/FVjDND1FjLGnmGMHdSf9eqh/IwZY/+gv5/3Msb+yrTfsiH1jFmC5LREPVO3MZRwzgf1PwDpAI4BmA4gC8AuAPOTPa9+utbzASwDsFco+zcA39VffxfAL/TXVwJ4DQADsArA5mTPP0H3YAKAZfrrkQAOA5g/nO6Dfi35+utMAJv1a3sKwK16+UMA/k5//TUAD+mvbwXwZLKvIUH34R4AjwN4WT8eVtdP/6J6rwyq34lEfc8BGA3guP63SH9dpJ/bAmC13uY1AJ/Sy6VjDNB1x/WZ1u/RLgDZAKbpzzzd7fmrxhig630EwJ366ywAhUP1GQMoBXACwAjhvn9xqD1jJEBOS+QzVY3heg0D9QHox4ewGsAbwvG9AO5N9rz68Xqn2t5whwBM0F9PAHBIf/17ALfJ6g2lfwBeBHDZcL0PAHIBbAdwDrQEKBl6ufm5APAGgNX66wy9Hkv23OO87kkA3gZwMYCX9S+9YXP99C/q98ug/p2I9XsOwG0Afi+U/14vmwDgoFBu1lONMQDXGPdn2v5cjXqq5+82xgBc7yhogjKzlQ/JZwxNMTgNTdjN0J/xuqH4jBGnnJbIZ6oaw23+Q8GUyHizGVTqZcOFcZzzagDQ/47Vy4f8fdG3FpdCWzEfVveBaVvuOwHUAXgL2kpJC+c8qFcRr9O8B/r5VgDFAzvjhPOfAL4DIKwfF2N4XT8RHYP2eyDO7zm38kpJOVzG6G8S8ZmO9j64jdHfTAdQD+B/mWY+9T+MsTwM0WfMOT8D4N8BnAJQDe2ZbcPQfsYGyXymUX/3DQXFgEnK+IDPIvUY0veFMZYP4FkA3+Sct7lVlZQN+vvAOQ9xzpdAW2VbCWCerJr+d0jdA8bY1QDqOOfbxGJJ1SF5/URMDMr3QAK+56ItTwoJ/EwPpvuQAc3k5EHO+VIAndBMQFQMpmtzoNu8XwfN/GcigDwAn5JUHUrP2IuBuJao2wwFxaASwGTheBKAqiTNJRnUMsYmAID+t04vH7L3hTGWCe3H8jHO+XN68bC7DwDAOW8B8C4028FCxliGfkq8TvMe6OcLADQN7EwTyrkArmWMVQB4AprpwX9i+Fw/ET2D7nsgQd9zbuWTJOVuY/QnifpMR3sfGlzG6G8qAVRyzjfrx89AUxSG6jO+FMAJznk957wPwHMA1mBoP2ODZD7TqL/7hoJisBXALN3rPAuak8r6JM9pIFkP4Hb99e3QbFGN8i/oHumrALQa20yDGd0D/48ADnDOfyWcGjb3gTFWwhgr1F+PgPaFewDARgA36dXs98C4NzcBeIfrxoaDEc75vZzzSZzzqdA+7+9wzv8Gw+T6iZgYVL8TCfyeewPA5YyxIn3F9nJo9tXVANoZY6v0sb4A+edFHKPfSOBnej2AW5kW0WYagFnQnDWlz19voxqjX+Gc1wA4zRiboxddAmA/hugzhmZCtIoxlqvPx7jeIfuMBZL5TKOXgfrTAWOg/kHzuj4Mzc76e8meTz9e51+h2eb1QdMC74BmP/c2gCP639F6XQbgt/o92QNgRbLnn6B7sBbaNthuADv1f1cOp/sAYBGAHfo92Avgn/Xy6dC+II8CeBpAtl6eox8f1c9PT/Y1JPBeXIhIBJNhd/30L6r3yqD5nUjk9xyAL+vv/aMAviSUr9C/P44B+A0iCU+lYwzgtcf1mQbwPf2aDkGP2OL2/FVjDNC1LgFQrj/nF6BFoBmyzxjAjwEc1Of0KLTIQkPqGSNBclqinqnbGKp/lPmYIAiCIAiCIIghYUpEEARBEARBEESckGJAEARBEARBEAQpBgRBEARBEARBkGJAEARBEARBEARIMSAIgiAIgiAIAqQYEARBEARBEAQBUgwIgiAIgiAIggApBgRBEARBEARBAPj/CpiwmAyTWXEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def moving_average(a, n=3) :\n",
    "    ret = np.cumsum(a, dtype=float)\n",
    "    ret[n:] = ret[n:] - ret[:-n]\n",
    "    return ret[n - 1:] / n\n",
    "\n",
    "def plot_training(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-100:])))\n",
    "    plt.plot(moving_average(rewards,20))\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss, average on 100 stpes')\n",
    "    plt.plot(moving_average(losses, 100),linewidth=0.2)\n",
    "    plt.show()\n",
    "\n",
    "plot_training(i, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
